001/******************************************************************************* 002 * This software is provided as a supplement to the authors' textbooks on digital 003 * image processing published by Springer-Verlag in various languages and editions. 004 * Permission to use and distribute this software is granted under the BSD 2-Clause 005 * "Simplified" License (see http://opensource.org/licenses/BSD-2-Clause). 006 * Copyright (c) 2006-2023 Wilhelm Burger, Mark J. Burge. All rights reserved. 007 * Visit https://imagingbook.com for additional details. 008 ******************************************************************************/ 009package Ch24_NonRigid_Matching; 010 011import ij.IJ; 012import ij.ImagePlus; 013import ij.gui.GenericDialog; 014import ij.gui.Roi; 015import ij.gui.ShapeRoi; 016import ij.plugin.filter.PlugInFilter; 017import ij.process.FloatProcessor; 018import ij.process.ImageProcessor; 019import imagingbook.common.color.sets.BasicAwtColor; 020import imagingbook.common.geometry.basic.Pnt2d; 021import imagingbook.common.geometry.basic.Pnt2d.PntDouble; 022import imagingbook.common.geometry.mappings.linear.ProjectiveMapping2D; 023import imagingbook.common.ij.DialogUtils; 024import imagingbook.common.ij.IjUtils; 025import imagingbook.common.ij.overlay.ColoredStroke; 026import imagingbook.common.ij.overlay.ShapeOverlayAdapter; 027import imagingbook.common.image.matching.lucaskanade.ImageExtractor; 028import imagingbook.common.image.matching.lucaskanade.LucasKanadeForwardMatcher; 029import imagingbook.common.image.matching.lucaskanade.LucasKanadeInverseMatcher; 030import imagingbook.common.image.matching.lucaskanade.LucasKanadeMatcher; 031import imagingbook.common.math.PrintPrecision; 032import imagingbook.core.jdoc.JavaDocHelp; 033import imagingbook.sampleimages.GeneralSampleImage; 034 035import java.awt.Color; 036import java.awt.Rectangle; 037import java.awt.geom.Path2D; 038import java.awt.geom.Rectangle2D; 039import java.util.Random; 040 041 042/** 043 * <p> 044 * This ImageJ plugin is a minimalistic example of using the Lucas-Kanade matchers. It performs a single test run on the 045 * current image, which is used as the <strong>search image I</strong>. A rectangular selection is required, which 046 * defines the initial mapping (Tinit) for the matcher, i.e., where it starts the search for the optimal match. 047 * </p> 048 * <p> 049 * The ROI rectangle is also used to extract the <strong>reference image R</strong> by random perturbation of its corner 050 * coordinates. The reference image R is extracted from this warped rectangle, whose corner coordinates are not known to 051 * the matcher. Given this warped template, the task of the Lucas-Kanade matcher is to find out from where in image I is 052 * was extracted and under which transformation, using only the pixel brightness information. Since all 4 corners of the 053 * ROI are perturbed, only a projective (4-point homography) transformation can recover an accurate match! 054 * </p> 055 * <p>The following steps are performed:</p> 056 * <ul> 057 * <li>Step 0: Create the search image I.</li> 058 * <li>Step 1: Get the rectangle of the ROI, create the (empty) reference image R of the same size.</li> 059 * <li>Step 2: Get the corner points (Q) of the ROI.</li> 060 * <li>Step 3: Perturb the ROI corners to form the quad QQ and use it to extract the reference image R from I.</li> 061 * <li>Step 4: Create the Lucas-Kanade matcher (forward or inverse).</li> 062 * <li>Step 5: Calculate the initial mapping Tinit from (centered) R → Q.</li> 063 * <li>Step 6: Calculate the real mapping from (centered) R → QQ (for validation only).</li> 064 * <li>Step 7: Initialize the matcher and run the matching loop.</li> 065 * <li>Step 8: Evaluate the results.</li> 066 * </ul> 067 * 068 * @author WB 069 * @version 2022/12/16 070 * @see LucasKanadeMatcher 071 * @see LucasKanadeForwardMatcher 072 * @see LucasKanadeInverseMatcher 073 * @see ImageExtractor 074 */ 075public class LucasKanade_Demo implements PlugInFilter, JavaDocHelp { 076 077 private static int MaxIterations = 100; 078 private static double PositionNoiseSigma = 2.5; 079 private static double PixelNoiseSigma = 0; // forward matcher: singular Hessian 080 private static boolean UseForwardMatcher = true; // inverse matcher has convergence problems! 081 private static boolean ShowReferenceImage = true; 082 private static boolean DrawBoundaries = true; 083 private static boolean ShowResultLog = true; 084 085 private static BasicAwtColor InitialQuadColor = BasicAwtColor.Green; 086 private static BasicAwtColor PerturbedQuadColor = BasicAwtColor.Blue; 087 private static BasicAwtColor FinalQuadColor = BasicAwtColor.Red; 088 089 static {PrintPrecision.set(6);} 090 091 private ImagePlus im; 092 093 /** 094 * Constructor, asks to open a predefined sample image if no other image is currently open. 095 */ 096 public LucasKanade_Demo() { 097 if (IjUtils.noCurrentImage()) { 098 if (DialogUtils.askForSampleImage(GeneralSampleImage.IrishManor)) { 099 ImagePlus imp = IJ.getImage(); 100 imp.setRoi(240, 90, 60, 60); 101 } 102 } 103 } 104 105 public int setup(String args, ImagePlus im) { 106 this.im = im; 107 return DOES_8G + ROI_REQUIRED; 108 } 109 110 public void run(ImageProcessor ip) { 111 Rectangle roi = ip.getRoi(); 112 if (roi == null) { 113 IJ.showMessage("Rectangular selection required!"); 114 return; 115 } 116 117 if (!runDialog()) { 118 return; 119 } 120 121 // Step 1: Create the search image I: 122 FloatProcessor I = ip.convertToFloatProcessor(); 123 124 // Step 2: Create the (empty) reference image R: 125 FloatProcessor R = new FloatProcessor(roi.width, roi.height); 126 127 // Step 3: Perturb the ROI corners to form the quad QQ and extract the reference image R: 128 Pnt2d[] Q = getCornerPoints(roi); // corners of the original quad 129 Pnt2d[] QQ = perturbGaussian(Q); // corners of the perturbed quad 130 new ImageExtractor(I).extractImage(R, QQ); 131 132 if (PixelNoiseSigma > 0) { // add Gaussian brightness noise to R 133 int wR = R.getWidth(); 134 int hR = R.getHeight(); 135 Random rg = new Random(); 136 for (int u = 0; u < wR; u++) { 137 for (int v = 0; v < hR; v++) { 138 double x = R.get(u, v) + rg.nextGaussian() * PixelNoiseSigma; 139 R.setf(u, v, (float) x); 140 } 141 } 142 } 143 144 if (ShowReferenceImage) { 145 new ImagePlus("R", R).show(); 146 } 147 148 // Step 4: Create the Lucas-Kanade matcher (forward or inverse): 149 LucasKanadeMatcher matcher = (UseForwardMatcher) ? 150 new LucasKanadeForwardMatcher(I, R) : 151 new LucasKanadeInverseMatcher(I, R); 152 153 // Step 5: Calculate the initial mapping Tinit from (centered) R -> Q: 154 ProjectiveMapping2D Tinit = matcher.getReferenceMappingTo(Q); 155 156 // Step 6: Calculate the real mapping from (centered) R -> QQ (for validation only): 157 ProjectiveMapping2D Treal = matcher.getReferenceMappingTo(QQ); 158 159 if (ShowResultLog) { 160 // IJ.log("Tinit = " + Matrix.toString(matcher.getParameters(Tinit))); 161 // IJ.log("Treal = " + Matrix.toString(matcher.getParameters(Treal))); 162 } 163 164 // -------------------------------------------------------------------------- 165 // Step 7: Initialize the matcher and run the matching loop: 166 ProjectiveMapping2D T = Tinit; 167 do { 168 T = matcher.iterateOnce(T); // returns null if iteration failed 169 int i = matcher.getIteration(); 170 double err = matcher.getRmsError(); 171 if (ShowResultLog) { 172 IJ.log(String.format("Iteration = %d, RMS error = %.2f", i, err)); 173 } 174 } 175 while (T != null && !matcher.hasConverged() && matcher.getIteration() < MaxIterations); 176 // -------------------------------------------------------------------------- 177 178 // quit if the matcher did not converge 179 if (T == null || !matcher.hasConverged()) { 180 IJ.log("no match found!"); 181 return; 182 } 183 184 // Step 8: Evaluate the results: 185 ProjectiveMapping2D Tfinal = T; 186 Pnt2d[] ptsFinal = Tfinal.applyTo(matcher.getReferencePoints()); 187 188 if (DrawBoundaries) { 189 ShapeOverlayAdapter ola = new ShapeOverlayAdapter(); 190 ola.setStroke(new ColoredStroke(0.2, InitialQuadColor.getColor())); 191 ola.addShape(makePolygon(Q)); 192 193 ola.setStroke(new ColoredStroke(0.5, PerturbedQuadColor.getColor())); 194 ola.addShape(makePolygon(QQ)); 195 196 ola.setStroke(new ColoredStroke(0.2, FinalQuadColor.getColor())); 197 ola.addShape(makePolygon(ptsFinal)); 198 im.setOverlay(ola.getOverlay()); 199 im.setRoi((Roi) null); 200 } 201 202 if (ShowResultLog) { 203 IJ.log(" ++++++++++++++++ Summary +++++++++++++++++++"); 204 // convert all mappings to projective (for comparison) 205 ProjectiveMapping2D TinitP = new ProjectiveMapping2D(Tinit); 206 ProjectiveMapping2D TrealP = new ProjectiveMapping2D(Treal); 207 ProjectiveMapping2D TfinalP = new ProjectiveMapping2D(Tfinal); 208 IJ.log("Matcher type: " + matcher.getClass().getSimpleName()); 209 IJ.log("Match found after " + matcher.getIteration() + " iterations."); 210 IJ.log("Final RMS error " + matcher.getRmsError()); 211 // IJ.log(" Tinit = " + Matrix.toString(matcher.getParameters(TinitP))); 212 // IJ.log(" Treal = " + Matrix.toString(matcher.getParameters(TrealP))); 213 // IJ.log(" Tfinal = " + Matrix.toString(matcher.getParameters(TfinalP))); 214 215 IJ.log("Corners of reference patch:"); 216 Pnt2d[] ptsRef = Treal.applyTo(matcher.getReferencePoints()); 217 for (Pnt2d pt : ptsRef) { 218 IJ.log(" pt = " + pt.toString()); 219 } 220 IJ.log("Corners for best match:"); 221 222 for (Pnt2d pt : ptsFinal) { 223 IJ.log(" pt = " + pt.toString()); 224 } 225 226 // Pnt2d test0 = PntInt.from(0, 0); 227 // IJ.log(" (0,0) by Treal = " + Treal.applyTo(test0).toString()); 228 // IJ.log(" (0,0) by Tfinal = " + Tfinal.applyTo(test0).toString()); 229 } 230 231 } 232 233 // ---------------------------------------------------------- 234 235 private Pnt2d[] getCornerPoints(Rectangle2D r) { 236 //IJ.log("getpoints2: r = " + r.toString()); 237 double x = r.getX(); 238 double y = r.getY(); 239 double w = r.getWidth(); 240 double h = r.getHeight(); 241 Pnt2d[] pts = new Pnt2d[4]; 242 pts[0] = PntDouble.from(x, y); 243 pts[1] = PntDouble.from(x + w - 1, y); // TODO: does -1 matter? YES - it seems WRONG!!! 244 pts[2] = PntDouble.from(x + w - 1, y + h - 1); 245 pts[3] = PntDouble.from(x, y + h - 1); 246 //IJ.log("getpoints2: p1-4 = " + pts[0] + ", " + pts[1] + ", " + pts[2] + ", " + pts[3]); 247 return pts; 248 } 249 250 private final Random rg = new Random(); 251 252 private Pnt2d perturbGaussian(Pnt2d p) { 253 double x = p.getX(); 254 double y = p.getY(); 255 x = x + rg.nextGaussian() * PositionNoiseSigma; 256 y = y + rg.nextGaussian() * PositionNoiseSigma; 257 return PntDouble.from(x, y); 258 } 259 260 private Pnt2d[] perturbGaussian(Pnt2d[] pntsIn) { 261 Pnt2d[] pntsOut = pntsIn.clone(); 262 for (int i = 0; i < pntsIn.length; i++) { 263 pntsOut[i] = perturbGaussian(pntsIn[i]); 264 } 265 return pntsOut; 266 } 267 268 @Deprecated 269 private Roi makePolygon(Pnt2d[] points, double strokeWidth, Color color) { 270 Path2D poly = new Path2D.Double(); 271 if (points.length > 0) { 272 poly.moveTo(points[0].getX(), points[0].getY()); 273 for (int i = 1; i < points.length; i++) { 274 poly.lineTo(points[i].getX(), points[i].getY()); 275 } 276 poly.closePath(); 277 } 278 Roi shapeRoi = new ShapeRoi(poly); 279 shapeRoi.setStrokeWidth(strokeWidth); 280 shapeRoi.setStrokeColor(color); 281 return shapeRoi; 282 } 283 284 285 private Path2D makePolygon(Pnt2d[] points) { 286 Path2D poly = new Path2D.Double(); 287 if (points.length > 0) { 288 poly.moveTo(points[0].getX(), points[0].getY()); 289 for (int i = 1; i < points.length; i++) { 290 poly.lineTo(points[i].getX(), points[i].getY()); 291 } 292 poly.closePath(); 293 } 294 return poly; 295 } 296 297 // ---------------------------------------------------------- 298 299 private boolean runDialog() { 300 GenericDialog gd = new GenericDialog(this.getClass().getSimpleName()); 301 gd.addHelp(getJavaDocUrl()); 302 gd.addNumericField("Maximum iterations", MaxIterations, 0); 303 gd.addNumericField("Position noise sigma", PositionNoiseSigma, 2); 304 // gd.addNumericField("Position noise sigma", PixelNoiseSigma, 2); 305 // gd.addCheckbox("Use forward matcher", UseForwardMatcher); 306 gd.addCheckbox("Show reference image", ShowReferenceImage); 307 gd.addCheckbox("Draw boundaries", DrawBoundaries); 308 gd.addCheckbox("Show result log", ShowResultLog); 309 310 gd.addEnumChoice("Initial quad color", InitialQuadColor); 311 gd.addEnumChoice("Perturbed quad color", PerturbedQuadColor); 312 gd.addEnumChoice("Final quad color", FinalQuadColor); 313 314 gd.showDialog(); 315 if (gd.wasCanceled()) 316 return false; 317 318 MaxIterations = (int) gd.getNextNumber(); 319 PositionNoiseSigma = gd.getNextNumber(); 320 // PixelNoiseSigma = gd.getNextNumber(); 321 // UseForwardMatcher = gd.getNextBoolean(); 322 ShowReferenceImage = gd.getNextBoolean(); 323 DrawBoundaries = gd.getNextBoolean(); 324 ShowResultLog = gd.getNextBoolean(); 325 326 InitialQuadColor = gd.getNextEnumChoice(BasicAwtColor.class); 327 PerturbedQuadColor = gd.getNextEnumChoice(BasicAwtColor.class); 328 FinalQuadColor = gd.getNextEnumChoice(BasicAwtColor.class); 329 330 return true; 331 } 332 333}