| Wheelchair Control Diploma Thesis, Author: Ondrej Vozda, FEEC BUT, 2013

- Main Page
- Packages
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Enumerations Enumerator Properties Events Pages
MyImageProcessingClasses.cs
1 ﻿using System;
2 using System.Collections.Generic;
3 using System.Drawing;
4 using System.Drawing.Imaging;
5 using System.IO;
6 using System.Linq;
7 using System.Runtime.Serialization.Formatters.Binary;
8 using System.Text;
9
10 //Emgu
11 using Emgu.CV;
12 using Emgu.CV.Features2D;
13 using Emgu.CV.Structure;
14 using Emgu.CV.UI;
15
16 namespace MyImageProcessingClasses
17 {
21 public static class ImageProcessingTools
22 {
23 #region Constants
24
28 public static readonly double HUE_HYSTERESIS = 10.0;
29
33 public static readonly double SATURATION_HYSTERESIS = 100.0;
34
38 public static readonly Byte ENCODER_PARAMETER_FIRST = 0;
39
43 private static readonly Int32 CENTER_RESTANGLE_SIZE = 15;
44
45 #endregion
46
47 #region Public objects
48
52 public static Font myFont = new Font(FontFamily.GenericSansSerif, 8f);
53
57 public static Brush[] myBrushes = { new SolidBrush(Color.GreenYellow),
58 new SolidBrush(Color.Red),
59 new SolidBrush(Color.Blue),
60 new SolidBrush(Color.Yellow)};
61
65 public static Point defaultPoint = new Point(0, 0);
66
70 public static StructuringElementEx StructEl_circle_5_5 = new StructuringElementEx(5, 5, 2, 2, Emgu.CV.CvEnum.CV_ELEMENT_SHAPE.CV_SHAPE_ELLIPSE);
71
75 public static ImageCodecInfo jpegCodec = GetEncoder(ImageFormat.Jpeg);
76
80 public static System.Drawing.Imaging.Encoder cameraFrameEncoder = System.Drawing.Imaging.Encoder.Quality;
81
85 public static EncoderParameters cameraFrameEncoderParameters = new EncoderParameters();
86
90 public static EncoderParameter[] cameraFrameEncoderParameter = CreateJpegEncoderParametersArray();
91
92 #endregion
93
94 #region Methods
95
100 private static EncoderParameter[] CreateJpegEncoderParametersArray()
101 {
102 const Byte PARAMETERS_COUNT = 101;
103
104 EncoderParameter[] eParams = new EncoderParameter[PARAMETERS_COUNT];
105
106 for (Byte i = 0; i < PARAMETERS_COUNT; i++)
107 eParams[i] = new EncoderParameter(cameraFrameEncoder, (Int64)i);
108
109 return eParams;
110 }
111
117 private static ImageCodecInfo GetEncoder(ImageFormat format)
118 {
119 ImageCodecInfo[] codecs = ImageCodecInfo.GetImageDecoders();
120
121 //find right encoder
122 foreach (ImageCodecInfo codec in codecs)
123 {
124 if (codec.FormatID == format.Guid)
125 {
126 return codec;
127 }
128 }
129
130 return null;
131 }
132
139 public static Image<Gray, Byte> ThresholdImageInHSV(Image<Bgr, Byte> image, Hsv thresholdHSV)
140 {
141 Image<Gray, Byte> imageOut;
142
143 //Threshold image in HSV color space
144 try
145 {
146 imageOut = image.Convert<Hsv, Byte>().InRange(
147 new Hsv(thresholdHSV.Hue - ImageProcessingTools.HUE_HYSTERESIS, thresholdHSV.Satuation - ImageProcessingTools.SATURATION_HYSTERESIS, 0), //Lower level
148 new Hsv(thresholdHSV.Hue + ImageProcessingTools.HUE_HYSTERESIS, thresholdHSV.Satuation + ImageProcessingTools.SATURATION_HYSTERESIS, 255) //Higher level
149 );
150 }
151 catch (Exception)
152 {
153 //Create empty image
154 imageOut = new Image<Gray, Byte>(image.Size);
155 }
156
157 //Morpohological operations to supress noise
158 imageOut = imageOut.MorphologyEx(ImageProcessingTools.StructEl_circle_5_5, Emgu.CV.CvEnum.CV_MORPH_OP.CV_MOP_OPEN, 2);
159
160 return imageOut;
161 }
162
168 public static Point FindCenter(Image<Gray, Byte> image)
169 {
170 Point center = new Point();
171
172 try
173 {
174 //Get all moments of the image, consider image to be binary
175 MCvMoments moments = image.GetMoments(true);
176
177 //Spatial moment 10 / area (moment 00) = x coordinate of center of mass
178 center.X = Convert.ToInt32(moments.m10 / moments.m00);
179
180 //Spatial moment 01 / area (moment 00) = y coordinate of center of mass
181 center.Y = Convert.ToInt32(moments.m01 / moments.m00);
182 }
183 catch (OverflowException)
184 {
185 //Reset the coordinates
186 center.X = 0;
187 center.Y = 0;
188 }
189
190 return center;
191 }
192
198 public static Rectangle GetCenterRectangle(Image<Gray, Byte> image)
199 {
200 Rectangle rect;
201 Size size = new Size(CENTER_RESTANGLE_SIZE, CENTER_RESTANGLE_SIZE);
202
203 //Find center of moment in specified image
204 Point center = FindCenter(image);
205
206 //Try to create the new rectangle
207 try
208 {
209 rect = new Rectangle(center.X - CENTER_RESTANGLE_SIZE / 2, center.Y - CENTER_RESTANGLE_SIZE / 2, CENTER_RESTANGLE_SIZE, CENTER_RESTANGLE_SIZE);
210 }
211 catch
212 {
213 rect = new Rectangle(new Point(0, 0), size);
214 }
215
216 return rect;
217 }
218
224 public static Rectangle GetCenterRectangle(Point point)
225 {
226 Rectangle rect;
227 Size size = new Size(CENTER_RESTANGLE_SIZE, CENTER_RESTANGLE_SIZE);
228
229 //Try to create the new rectangle
230 try
231 {
232 rect = new Rectangle(point.X - CENTER_RESTANGLE_SIZE / 2, point.Y - CENTER_RESTANGLE_SIZE / 2, CENTER_RESTANGLE_SIZE, CENTER_RESTANGLE_SIZE);
233 }
234 catch
235 {
236 rect = new Rectangle(new Point(0, 0), size);
237 }
238
239 return rect;
240 }
241
247 public static Point Center(this Rectangle rect)
248 {
249 return new Point(rect.Left + rect.Width / 2, rect.Top + rect.Height / 2);
250 }
251
252 #endregion
253 }
254
258 public class CameraCalibrator
259 {
263 public enum PointLocation : byte
264 {
268 UPPER_LEFT = 0,
269
273 UPPER_RIGHT = 1,
274
278 LOWER_RIGHT = 2,
279
283 LOWER_LEFT = 3
284 };
285
289 public static Dictionary<Byte, PointF> CalibrationPoints = new Dictionary<Byte, PointF>()
290 {
291 {(Byte)PointLocation.UPPER_LEFT, new PointF(-0.5f,1f)},
292 {(Byte)PointLocation.UPPER_RIGHT, new PointF(0.5f,1f)},
293 {(Byte)PointLocation.LOWER_RIGHT, new PointF(0.5f,0.5f)},
294 {(Byte)PointLocation.LOWER_LEFT, new PointF(-0.5f,0.5f)}
295 };
296
300 private List<PointF> templatePoints;
301
305 private List<PointF> selectedPoints;
306
310 private HomographyMatrix homographyMatrix;
311
315 public HomographyMatrix HomographyMat
316 {
317 get
318 {
319 return homographyMatrix;
320 }
321 }
322
326 public CameraCalibrator()
327 {
328 //Template points initialization
329 templatePoints = new List<PointF> { CalibrationPoints[(Byte)PointLocation.UPPER_LEFT],
330 CalibrationPoints[(Byte)PointLocation.UPPER_RIGHT],
331 CalibrationPoints[(Byte)PointLocation.LOWER_RIGHT],
332 CalibrationPoints[(Byte)PointLocation.LOWER_LEFT],
333 };
334
335 //Selected point are initalized as empty set
336 selectedPoints = new List<PointF> { new PointF(-1f, -1f),
337 new PointF(-1f, -1f),
338 new PointF(-1f, -1f),
339 new PointF(-1f, -1f)};
340
341 homographyMatrix = new HomographyMatrix();
342 }
343
349 public void AddCalibrationPoint(PointF point, PointLocation location)
350 {
351 selectedPoints[(int)location] = point;
352 }
353
357 public void CalculateHomography()
358 {
359 homographyMatrix = CameraCalibration.FindHomography(selectedPoints.ToArray(),
360 templatePoints.ToArray(),
361 Emgu.CV.CvEnum.HOMOGRAPHY_METHOD.DEFAULT, 1.0);
362 }
363
370 public PointF TransformPoint(PointF point)
371 {
372 //Create data array
373 Double[,] data = new Double[3, 1]{ {(Double)point.X}, //X
374 {(Double)point.Y}, //Y
375 {1.0} //Scale
376 };
377
378 //Matrix from data
379 Matrix<Double> matrixTemp = new Matrix<double>(data);
380
381 //Transform vector using homography matrix
382 matrixTemp = homographyMatrix.Mul(matrixTemp);
383
384 /*Return transformed point.
385 Note that homographyMatrix is not generally the affine homography matrix,
386 therefore x2 = x2'/w' and y2 = y2'/w', where w' is scale in the new plane*/
387 return new PointF((float)(matrixTemp[0, 0] / matrixTemp[2, 0]), (float)(matrixTemp[1, 0] / matrixTemp[2, 0]));
388 }
389
394 public Boolean AllCalibrationPointsSpecified
395 {
396 get
397 {
398 foreach (PointF point in selectedPoints)
399 if (point.X < 0 || point.Y < 0)
400 return false;
401
402 return true;
403 }
404 }
405
410 public override string ToString()
411 {
412 StringBuilder outString = new StringBuilder();
413
414 //Parse matrix elements into string
415 for (byte i = 0; i < homographyMatrix.Rows; i++)
416 {
417 for (byte j = 0; j < homographyMatrix.Cols; j++)
418 {
419 outString.Append(homographyMatrix[i,j].ToString("F3") + " ");
420 }
421
422 outString.Append(Environment.NewLine);
423 }
424
425 return outString.ToString();
426 }
427 }
428
432 public class FPS_Counter
433 {
437 private DateTime lastFrameTime;
438
442 private double fps;
443
447 public double FPS
448 {
449 get
450 {
451 return fps;
452 }
453 }
454
458 public FPS_Counter()
459 {
460 //Set default values
461 lastFrameTime = DateTime.Now;
462 fps = -1.0;
463 }
464
468 public void FrameCaptured()
469 {
470 DateTime currentTime = DateTime.Now;
471
472 //Calculate fps
473 fps = Convert.ToDouble(1 / ((currentTime - lastFrameTime).Milliseconds / 1000.0));
474
475 lastFrameTime = currentTime;
476 }
477 }
478
482 public class MarksDetector
483 {
487 private const Byte MARK_BUFFER_LENGTH = 5;
488
492 private const float DEVIATION_THRESH = 10f;
493
497 private Queue<PointF> marksBuffer = new Queue<PointF>(MARK_BUFFER_LENGTH);
498
502 private Object bufferAccessLock = new Object();
503
507 public MarksDetector()
508 { }
509
513 public void Reset()
514 {
515 lock(bufferAccessLock)
516 marksBuffer.Clear();
517 }
518
523 public void AddPoint(PointF point)
524 {
525 lock (bufferAccessLock)
526 {
527 if (marksBuffer.Count < MARK_BUFFER_LENGTH)
528 {
529 //Buffer not full, just add point
530 marksBuffer.Enqueue(point);
531 }
532 else
533 {
534 //Dequeue oldest point and and new one
535 marksBuffer.Dequeue();
536 marksBuffer.Enqueue(point);
537 }
538 }
539 }
540
544 public Boolean IsBufferFull
545 {
546 get
547 {
548 lock(bufferAccessLock)
549 return (marksBuffer.Count == MARK_BUFFER_LENGTH);
550 }
551 }
552
558 public Boolean DetectMark(out Point detectedPoint)
559 {
560 List<float> xCoordList = new List<float>(MARK_BUFFER_LENGTH);
561 List<float> yCoordList = new List<float>(MARK_BUFFER_LENGTH);
562
563 Double averageX, averageY, squaresSum, deviation; //Variables for statictics
564
565 detectedPoint = new Point();
566
567 //Lock buffer access to get consistent data
568 lock(bufferAccessLock)
569 {
570 //Buffer not full, can not calculate average
571 if (!IsBufferFull)
572 return false;
573
574 foreach (PointF point in marksBuffer)
575 {
576 xCoordList.Add(point.X);
577 yCoordList.Add(point.Y);
578 }
579 }
580
581 //Calculate standard deviation of x coordinates measurements
582 averageX = xCoordList.Average();
583 squaresSum = xCoordList.Select(val => (val - averageX) * (val - averageX)).Sum();
584 deviation = Math.Sqrt(squaresSum / xCoordList.Count);
585
586 if (deviation > DEVIATION_THRESH)
587 return false;
588
589 //Calculate standard deviation of y coordinates measurements
590 averageY = yCoordList.Average();
591 squaresSum = yCoordList.Select(val => (val - averageY) * (val - averageY)).Sum();
592 deviation = Math.Sqrt(squaresSum / yCoordList.Count);
593
594 if (deviation > DEVIATION_THRESH)
595 return false;
596
597 //Deviation within allowed border, return point
598 detectedPoint = new Point((Int32)averageX, (Int32)averageY);
599
600 return true;
601 }
602 }
603
608 public class FeatureDetector : Emgu.Util.DisposableObject
609 {
610 #region Private variables
611
612 private Features2DTracker<float> _tracker;
613 private SURFDetector _detector;
614 private MemStorage _rectangleStorage;
615 private Contour<Point> _rectangle;
616 private Byte _referenceHue;
617
618 #endregion
619
623 public enum RefrenceHue : byte
624 {
628 Red = 5,
629
633 Green = 70,
634
638 Blue = 160,
639
643 Cyan = 120,
644
648 Magenta = 200,
649
653 Yellow = 40
654 };
655
661 public FeatureDetector(Image<Bgr, Byte> featureModel, Byte referenceHue)
662 {
663 _detector = new SURFDetector(500, false);
664 _referenceHue = referenceHue;
665
666 using (Image<Gray, Byte> mask = GetPixelMask(featureModel, referenceHue))
667 {
668 try
669 {
670 //Try to detect features
671 _tracker = new Features2DTracker<float>(_detector.DetectFeatures(mask, null));
672 }
673 catch (IndexOutOfRangeException)
674 {
675 //No features detected
676 }
677 }
678 _rectangleStorage = new MemStorage();
679 _rectangle = new Contour<Point>(_rectangleStorage);
680
681 //Create new contour reference rectangle
682 _rectangle.PushMulti(new Point[] {
683 new Point(0, 0),
684 new Point(3, 0),
685 new Point(3, 2),
686 new Point(0, 2)},
687 Emgu.CV.CvEnum.BACK_OR_FRONT.FRONT);
688 }
689
696 private static Image<Gray, Byte> GetPixelMask(Image<Bgr, byte> image, Byte referenceHue)
697 {
698 const Byte HUE_RANGE = 25; //Allowed hue deviation
699 const Double VALUE_LOWER = 30.0; //Lower threshold of value
700 const Double VALUE_HIGHER = 230.0; //Higher threshold of value
701
702 using (Image<Hsv, Byte> hsv = image.Convert<Hsv, Byte>())
703 {
704 Image<Gray, Byte>[] channels = hsv.Split();
705
706 try
707 {
708 //channels[0] is hue channel, select pixels within the specified range
709 CvInvoke.cvInRangeS(channels[0],
710 new MCvScalar((referenceHue - HUE_RANGE) < 1 ? 1.0 : (Double)(referenceHue - HUE_RANGE)),
711 new MCvScalar((referenceHue + HUE_RANGE) > 255 ? 255.0 : (Double)(referenceHue + HUE_RANGE)),
712 channels[0]);
713
714 //Threshold saturation
715 CvInvoke.cvThreshold(channels[1], channels[1], Byte.MaxValue / 2.0, (Double)Byte.MaxValue, Emgu.CV.CvEnum.THRESH.CV_THRESH_BINARY);
716
717 //channels[2] (value) -> filter value to eliminate black and white pixels
718 CvInvoke.cvInRangeS(channels[2], new MCvScalar(VALUE_LOWER), new MCvScalar(VALUE_HIGHER), channels[2]);
719 channels[2]._Not();
720
721 //Create mask
722 CvInvoke.cvAnd(channels[0], channels[2], channels[0], IntPtr.Zero);
723 CvInvoke.cvAnd(channels[0], channels[1], channels[0], IntPtr.Zero);
724 }
725 finally
726 {
727 channels[1].Dispose();
728 channels[2].Dispose();
729 }
730
731 return channels[0];
732 }
733 }
734
735 private void FindFeature(Image<Bgr, byte> img, List<Image<Gray, Byte>> featureList, List<Rectangle> boxList, Contour<Point> contours)
736 {
737 Int32 maxFeatures = 0;
738
739 //Run through all detected contours
740 for (; contours != null; contours = contours.HNext)
741 {
742 contours.ApproxPoly(contours.Perimeter * 0.02, 0, contours.Storage);
743 if (contours.Area > 50)
744 {
745 double ratio = CvInvoke.cvMatchShapes(_rectangle, contours, Emgu.CV.CvEnum.CONTOURS_MATCH_TYPE.CV_CONTOURS_MATCH_I3, 0);
746
747 if (ratio > 0.3) //Not a good match of contour shape, re-run
748 {
749 Contour<Point> child = contours.VNext;
750 if (child != null)
751 FindFeature(img, featureList, boxList, child);
752 continue;
753 }
754
755 Rectangle box = contours.BoundingRectangle;
756
757 Image<Gray, Byte> candidate;
758 using (Image<Bgr, Byte> tmp = img.Copy(box))
759 candidate = tmp.Convert<Gray, byte>();
760
761 //Set the value of pixels out of the contour region to zero
762 using (Image<Gray, Byte> mask = new Image<Gray, byte>(box.Size))
763 {
764 mask.Draw(contours, new Gray(255), new Gray(255), 0, -1, new Point(-box.X, -box.Y));
765
766 double mean = CvInvoke.cvAvg(candidate, mask).v0;
767 candidate._ThresholdBinary(new Gray(mean), new Gray(255.0));
768 candidate._Not();
769 mask._Not();
770 candidate.SetValue(0, mask);
771 }
772
773 //Detect features
774 ImageFeature<float>[] features = _detector.DetectFeatures(candidate, null);
775
776 //Match features of image and model
777 Features2DTracker<float>.MatchedImageFeature[] matchedFeatures = new Features2DTracker<float>.MatchedImageFeature[0];
778
779 //At least two features neccessary to find neighbours
780 if (features.Length >= 2)
781 matchedFeatures = _tracker.MatchFeature(features, 2);
782
783 int goodMatchCount = 0;
784
785 //Compare metric distance of detected features to find number of corresponding features
786 foreach (Features2DTracker<float>.MatchedImageFeature ms in matchedFeatures)
787 if (ms.SimilarFeatures[0].Distance < 0.7) goodMatchCount++;
788
789 if ((goodMatchCount >= 3) && (goodMatchCount > maxFeatures))
790 {
791 //Add only the item with highest count of matching features
792 maxFeatures = goodMatchCount;
793
794 boxList.Clear();
795 boxList.Add(box);
796
797 featureList.Clear();
798 featureList.Add(candidate);
799 }
800
801 }
802 }
803 }
804
811 public void DetectFeature(Image<Bgr, byte> img, List<Image<Gray, Byte>> featureList, List<Rectangle> boxList)
812 {
813 Image<Bgr, Byte> smoothImg = img.SmoothGaussian(5, 5, 1.5, 1.5);
814 Image<Gray, Byte> smoothedMask = GetPixelMask(smoothImg, _referenceHue);
815
816 //Use Dilate followed by Erode to eliminate small gaps in some countour.
817 smoothedMask._Dilate(2);
818 smoothedMask._Erode(1);
819
820 using (Image<Gray, Byte> canny = smoothedMask.Canny(100.0, 50.0))
821 using (MemStorage stor = new MemStorage())
822 {
823 Contour<Point> contours = canny.FindContours(
824 Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
825 Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_TREE,
826 stor);
827
828 try
829 { FindFeature(img, featureList, boxList, contours); }
830 catch (NullReferenceException) { ; }
831 }
832 }
833
837 protected override void DisposeObject()
838 {
839 try
840 {
841 _tracker.Dispose();
842 _rectangleStorage.Dispose();
843 }
844 catch (NullReferenceException)
845 { /*Object instance not available, nothing to dipose*/ }
846 }
847 }
848 }