Thursday, June 7, 2012

Digit Recognition with OpenCV (CPP Version)

This code is basically a cpp port inspired by the article "Simple Digit Recognition OCR in OpenCV-Python" . I even shamelessly stole his images. The handy thing of this example is the self test loop. This assures your thing is really working and ready to go on real world images.

The code has currently just on picture per digit to learn from and achieves an accuracy of 97% on the big image. To improve it you could add more samples for each number.

I tried to use as much the new cpp api as possible. At some point I still rely on the old c api. I am still trying to replace CvMat with cv::Mat and especially this loop

for (int n = 0; n < ImageSize; n++)
{
   trainData->data.fl[i * ImageSize + n] = outfile.data[n];
}

From my believe this is not the way things should be handled in an API but so far I found no other way.
Take the following images by clicking on them and downloading them to your images folder, which is in my case ../images relative to the location of the executable.


0.png
1.png
2.png
3.png
4.png
5.png
6.png
7.png
8.png
9.png
buchstaben.png
#include "opencv2/ml/ml.hpp">
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include <iostream>
#include <stdio.h>

using namespace cv;
using namespace std;

const int train_samples = 1;
const int classes = 10;
const int sizex = 20;
const int sizey = 30;
const int ImageSize = sizex * sizey;
char pathToImages[] = "../images";

void PreProcessImage(Mat *inImage,Mat *outImage,int sizex, int sizey);
void LearnFromImages(CvMat* trainData, CvMat* trainClasses);
void RunSelfTest(KNearest& knn2);
void AnalyseImage(KNearest knearest);
/** @function main */
int main(int argc, char** argv)
{

 CvMat* trainData = cvCreateMat(classes * train_samples,ImageSize, CV_32FC1);
 CvMat* trainClasses = cvCreateMat(classes * train_samples, 1, CV_32FC1);

 namedWindow("single", CV_WINDOW_AUTOSIZE);
 namedWindow("all",CV_WINDOW_AUTOSIZE);

 LearnFromImages(trainData, trainClasses);

 KNearest knearest(trainData, trainClasses);

 RunSelfTest(knearest);

 cout << "losgehts\n";

 AnalyseImage(knearest);

 return 0;

}

void PreProcessImage(Mat *inImage,Mat *outImage,int sizex, int sizey)
{
 Mat grayImage,blurredImage,thresholdImage,contourImage,regionOfInterest;

 vector<vector<Point> > contours;

 cvtColor(*inImage,grayImage , COLOR_BGR2GRAY);

 GaussianBlur(grayImage, blurredImage, Size(5, 5), 2, 2);
 adaptiveThreshold(blurredImage, thresholdImage, 255, 1, 1, 11, 2);

 thresholdImage.copyTo(contourImage);

 findContours(contourImage, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);

 int idx = 0;
 size_t area = 0;
 for (size_t i = 0; i < contours.size(); i++)
 {
  if (area < contours[i].size() )
  {
   idx = i;
   area = contours[i].size();
  }
 }

 Rect rec = boundingRect(contours[idx]);

 regionOfInterest = thresholdImage(rec);

 resize(regionOfInterest,*outImage, Size(sizex, sizey));

}

void LearnFromImages(CvMat* trainData, CvMat* trainClasses)
{
 Mat img;
 char file[255];
 for (int i = 0; i < classes; i++)
 {
  sprintf(file, "%s/%d.png", pathToImages, i);
  img = imread(file, 1);
  if (!img.data)
  {
    cout << "File " << file << " not found\n";
    exit(1);
  }
  Mat outfile;
  PreProcessImage(&img, &outfile, sizex, sizey);
  for (int n = 0; n < ImageSize; n++)
  {
   trainData->data.fl[i * ImageSize + n] = outfile.data[n];
  }
  trainClasses->data.fl[i] = i;
 }

}

void RunSelfTest(KNearest& knn2)
{
 Mat img;
 CvMat* sample2 = cvCreateMat(1, ImageSize, CV_32FC1);
 // SelfTest
 char file[255];
 int z = 0;
 while (z++ < 10)
 {
  int iSecret = rand() % 10;
  //cout << iSecret;
  sprintf(file, "%s/%d.png", pathToImages, iSecret);
  img = imread(file, 1);
  Mat stagedImage;
  PreProcessImage(&img, &stagedImage, sizex, sizey);
  for (int n = 0; n < ImageSize; n++)
  {
   sample2->data.fl[n] = stagedImage.data[n];
  }
  float detectedClass = knn2.find_nearest(sample2, 1);
  if (iSecret != (int) ((detectedClass)))
  {
   cout << "Falsch. Ist " << iSecret << " aber geraten ist "
     << (int) ((detectedClass));
   exit(1);
  }
  cout << "Richtig " << (int) ((detectedClass)) << "\n";
  imshow("single", img);
  waitKey(0);
 }

}

void AnalyseImage(KNearest knearest)
{

 CvMat* sample2 = cvCreateMat(1, ImageSize, CV_32FC1);

 Mat image, gray, blur, thresh;

 vector < vector<Point> > contours;
 image = imread("../images/buchstaben.png", 1);

 cvtColor(image, gray, COLOR_BGR2GRAY);
 GaussianBlur(gray, blur, Size(5, 5), 2, 2);
 adaptiveThreshold(blur, thresh, 255, 1, 1, 11, 2);
 findContours(thresh, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);

 for (size_t i = 0; i < contours.size(); i++)
 {
  vector < Point > cnt = contours[i];
  if (contourArea(cnt) > 50)
  {
   Rect rec = boundingRect(cnt);
   if (rec.height > 28)
   {
    Mat roi = image(rec);
    Mat stagedImage;
    PreProcessImage(&roi, &stagedImage, sizex, sizey);
    for (int n = 0; n < ImageSize; n++)
    {
     sample2->data.fl[n] = stagedImage.data[n];
    }
    float result = knearest.find_nearest(sample2, 1);
    rectangle(image, Point(rec.x, rec.y),
      Point(rec.x + rec.width, rec.y + rec.height),
      Scalar(0, 0, 255), 2);

    imshow("all", image);
    cout << result << "\n";

    imshow("single", stagedImage);
    waitKey(0);
   }

  }

 }
}

Other Links:

K-Nearest Neighbors in OpenCV

digit recogntion using OpenCV on android

 

25 comments:

Unknown said...

please tell me the version of OpenCV that you used in this project. thanks

Anonymous said...

That was 2.4.1

陳柏元 said...

Thank you for you good article. Now I have 3 samples for each digit, what should I do to trainData and trainClasses ?

Unknown said...

The method LearnFromImages in the above code sample does this for you

陳柏元 said...

Hi Tobias,
If I have twenty sample images, so train_samples =2. But there is no train_samples in LearnFromImages, trainData and trainClasses won't change. Is this correct ? Or what should we do ?

Unknown said...

Hi, you wrote "I tried to use as much the new cpp api as possible. At some point I still rely on the old c api. I am still trying to replace CvMat with cv::Mat and especially this loop "

i would do that like this:

Mat sample2 = Mat(1, ImageSize, CV_32FC1);
sample2 = stagedImage.reshape(1,1);
sample2.convertTo(sample2, CV_32FC1);

Un das wars ;)

Unknown said...

Hi,
I create my image folder on my desktop
in LearnFromImage(*,*) i code as your code bellow
but i receive an output result : "File /Users/myUser/Desktop/images/0.png not found

and my pathToImages :char pathToImages[] = "/Users/myUser/Desktop/images";

so , what does it mean ? and how can i fix it ?

thank !

elLoco said...

There are some images in the above post marked as 0.png, 1.png...

Download them and put them to the proper location

rebeccacarf said...

Hi,
your program is great, but it sometimes detects number 0 as number 3 in the big picture. Do you know about it and would you be able to "repair" it? I would be very thankful.

Anonymous said...

hy, the void PreProcessImage for what is, what is doing that? becouse i have some problem, i have to click break ad the line where is the problem this is"Rect rec = boundingRect(contours[idx]);"

Anonymous said...

Hello, i have this problem, can you help me? "Unhandled exception at 0x006A9071 (opencv_imgproc243d.dll) in opencvkep.exe: 0xC0000005: Access violation reading location 0x00CA1000. " Thanks.

Anonymous said...

Hey thanks for good work, are YOu using A neural network for this ?

Unknown said...

where am i put 0.jpg,1.jpg?

what is mean from /images?

must i create folder with name images?

Anonymous said...

By the way, most of people ask for the images folder directory. You should put the images folder to directory which is also include .sln

Unknown said...

Hi,

What do you think is the best way to improve the accuracy?

How much of training set will be enough?

In terms of training set, what font in different size will make be good or something else ?

Could you please give me some direction ?

Thank you very much.


Anonymous said...

Hi,
At the moment I’m struggling with porting the sample to OpenCV 3.1.
...
Mat_ trainData(classes * train_samples, gImageSize, CV_32FC1);
Mat_ trainClasses(classes * train_samples, 1, CV_32FC1);
...
Ptr kNearest = ml::KNearest::create();
kNearest->setIsClassifier(true);
kNearest->setDefaultK(10);
kNearest->setAlgorithmType(cv::ml::KNearest::Types::BRUTE_FORCE);

Ptr trainingData = ml::TrainData::create(trainData, ml::SampleTypes::ROW_SAMPLE, trainClasses);
kNearest->train(trainingData, 0);
...
Mat_ matResults;
float detectedClass = kNearest->findNearest(sample, 1, matResults);

Till now I was not able to detect any number. I also changed the algorithm type to KDTree without any success.

So what is the missing point?

Thanks in advance!

Unknown said...

I am stoned???
main.cpp:19: error: ‘KNearest’ was not declared in this scope

Anonymous said...

hi. sorry, but what language is it?

Phuwadon said...

รีวิวสล็อต Roma Legacy เกมสล็อตยอดฮิต จากค่ายเกมสุดคลาสสิค JOKER GAME SLOT ที่เป็นกระแสการทำเงินอย่างยาวนาน โบนัสเงินรางวัลสูง

Phuwadon said...

PG Slot เป็นเกมสล็อตที่ร้อนแรงและมีส่วนร่วมมากที่สุดในยุคนี้ ทดลองเล่นสล็อต https://www.pgslot168game.com/pgslot-demo/ เป็นเว็บไซต์ที่มีโปรโมชั่นเป็นของตัวเอง ฝาก 10 ที่เดียวก็สามารถลุ้นได้ถึง 100 คะแนน นอกจากนี้ การฝากทุกครั้งยังมาพร้อมกับโบนัสอีกด้วย รับ 20% ทั้งวัน 24 ชม. ไม่มีหนี้แน่นอน

เกมที่น่าระทึกใจ said...

สล็อต marine88 ยินดีต้อนรับสู่ pg slot คาสิโนออนไลน์!ตรงนี้คุณจะได้เจอกับเกมที่น่าระทึกใจที่จะทำให้ท่านเพลิดเพลินเจริญใจได้นานหลายชั่วโมงไม่ว่าจะเป็นสล็อตเกมโต๊ะหรือคาสิโน

ตื่นเต้นกับประสบการณ์ said...

เล่นเกม PG Piggy สล็อต ตื่นเต้นที่สุดของเกมสล็อตเล่นและพบกับบทความที่ครอบคลุมทุกสิ่งที่คุณต้องรู้เกี่ยวกับเกมนี้ PG SLOT ร่วมสนุกกับหมูตัวน่ารักในการตามหาความมั่งคั่งและมีเวลาที่น่าตื่นเต้นกับประสบการณ์

PG siam66 said...

siam66 มีเกมให้เล่นมากไม่น้อยเลยทีเดียวคาสิโน สล็อตออนไลน์ รูเล็ต บาคาร่า ยิงปลา พนันบอลแทงกีฬามากกว่า 1000 แมตซ์ pgslot มีโปรโมชันเยอะแยะให้เลือกยิ่งแทงมากยิ่งได้กลับคืน

PG Online said...

ทางเข้าเล่นเกม appg pg slot ในโลกของเกมออนไลน์ที่เติบโตอย่างรวดเร็ว เว็บไซต์ appg pg slot กลายเป็นแพลตฟอร์มที่ได้รับความนิยมสำหรับผู้ที่หลงใหลในการเล่นสล็อต PGSLOT

PG SLOT said...

bacc9999 ผู้ให้บริการเกมสล็อตออนไลน์บนโทรศัพท์เคลื่อนที่ที่มีเกมนานัปการให้เลือก pg slot เป็นเกมรูปแบบใหม่ที่ทำเงินให้ผู้เล่นได้เงินจริง มีแนวทางสอนการเล่นเกมสล็อตออนไลน์