001/*******************************************************************************
002 * Permission to use and distribute this software is granted under the BSD 2-Clause
003 * "Simplified" License (see http://opensource.org/licenses/BSD-2-Clause).
004 * Copyright (c) 2016-2023 Wilhelm Burger. All rights reserved.
005 * Visit https://imagingbook.com for additional details.
006 ******************************************************************************/
007package imagingbook.calibration.zhang;
008
009import imagingbook.common.geometry.basic.Pnt2d;
010import org.apache.commons.math3.analysis.MultivariateMatrixFunction;
011import org.apache.commons.math3.analysis.MultivariateVectorFunction;
012import org.apache.commons.math3.fitting.leastsquares.LeastSquaresFactory;
013import org.apache.commons.math3.fitting.leastsquares.LeastSquaresOptimizer.Optimum;
014import org.apache.commons.math3.fitting.leastsquares.LevenbergMarquardtOptimizer;
015import org.apache.commons.math3.fitting.leastsquares.MultivariateJacobianFunction;
016import org.apache.commons.math3.linear.ArrayRealVector;
017import org.apache.commons.math3.linear.RealVector;
018
019import java.util.Arrays;
020
021/**
022 * Abstract super-class for non-linear optimizers used for final, overall optimization of calibration parameters. The
023 * actual optimization is performed by the sub-classes.
024 *
025 * @author WB
026 */
027public abstract class NonlinearOptimizer {
028
029        private static int maxEvaluations = 1000;
030        private static int maxIterations = 1000;
031
032        final Pnt2d[] modelPts;
033        final Pnt2d[][] obsPts;
034        final int M;        // number of views
035        final int N;        // number of model points
036        int camParLength;        // number of camera parameters (7)
037        int viewParLength;    // number of view parameters (6)
038
039        private Camera initCam = null;
040        private Camera finalCamera = null;
041        private ViewTransform[] initViews = null;
042        private ViewTransform[] finalViews = null;
043
044        NonlinearOptimizer(Pnt2d[] modelPts, Pnt2d[][] obsPts) {
045                this.modelPts = modelPts;
046                this.obsPts = obsPts;
047                this.M = obsPts.length;
048                this.N = modelPts.length;
049        }
050
051        /**
052         * Performs Levenberg-Marquardt non-linear optimization to get better estimates of the parameters.
053         *
054         * @param initCam the initial camera parameters
055         * @param initViews the initial view transforms
056         */
057        void optimize(Camera initCam, ViewTransform[] initViews) {
058                this.initCam = initCam;
059                this.initViews = initViews;
060                this.camParLength = initCam.getParameterVector().length;
061                this.viewParLength = initViews[0].getParameters().length;
062
063                MultivariateVectorFunction V = makeValueFun();
064                MultivariateMatrixFunction J = makeJacobianFun();
065
066                RealVector start = makeInitialParameters();
067                RealVector observed = makeObservedVector();
068
069                MultivariateJacobianFunction model = LeastSquaresFactory.model(V, J);
070                LevenbergMarquardtOptimizer lm = new LevenbergMarquardtOptimizer();
071                Optimum result = lm.optimize(LeastSquaresFactory.create(
072                                model,
073                                observed,
074                                start,
075                                null,
076                                maxEvaluations,
077                                maxIterations));
078
079//              System.out.println(NonlinearOptimizer.class.getSimpleName() + "; iterations = " + result.getIterations());
080                updateEstimates(result.getPoint());
081        }
082
083        /**
084         * To be implemented by subclasses.
085         *
086         * @return a vector value function
087         */
088        abstract MultivariateVectorFunction makeValueFun();
089
090        /**
091         * To be implemented by subclasses.
092         *
093         * @return a Jacobian function
094         */
095        abstract MultivariateMatrixFunction makeJacobianFun();
096
097
098        /**
099         * Common value function for optimizers defined in sub-classes.
100         */
101        class ValueFun implements MultivariateVectorFunction {
102
103                @Override
104                public double[] value(double[] params) {
105                        final double[] a = Arrays.copyOfRange(params, 0, camParLength);
106                        final Camera cam = new Camera(a);
107                        final double[] Y = new double[2 * M * N];
108                        int c = 0;
109                        for (int m = 0; m < M; m++) {
110                                int q = camParLength + m * viewParLength;
111                                double[] w = Arrays.copyOfRange(params, q, q + viewParLength);
112                                ViewTransform view = new ViewTransform(w);
113                                for (int n = 0; n < N; n++) {
114                                        double[] uv = cam.project(view, modelPts[n]);
115                                        Y[c * 2 + 0] = uv[0];
116                                        Y[c * 2 + 1] = uv[1];
117                                        c = c + 1;
118                                }
119                        }
120                        return Y;
121                }
122        }
123
124        // ---------------------------------------------------------------------
125
126        private RealVector makeInitialParameters() {
127                double[] s = initCam.getParameterVector();
128                double[] c = new double[s.length + M * viewParLength];
129
130                // insert camera parameters at beginning of c
131                System.arraycopy(s, 0, c, 0, s.length);
132
133                // insert M view parameters
134                int start = s.length;
135                for (int i = 0; i < M; i++) {
136                        double[] w = initViews[i].getParameters();
137                        System.arraycopy(w, 0, c, start, w.length);
138                        start = start + w.length;
139                }
140                return new ArrayRealVector(c);
141        }
142
143
144        /**
145         * Stack the observed image coordinates of the calibration pattern points into a vector.
146         *
147         * @return the observed vector
148         */
149        RealVector makeObservedVector() {
150                double[] obs = new double[M * N * 2];
151                for (int i = 0, k = 0; i < M; i++) {
152                        for (int j = 0; j < N; j++, k++) {
153                                obs[k * 2 + 0] = obsPts[i][j].getX();
154                                obs[k * 2 + 1] = obsPts[i][j].getY();
155                        }
156                }
157                // obs = [u_{0,0}, v_{0,0}, u_{0,1}, v_{0,1}, ..., u_{M-1,N-1}, v_{M-1,N-1}]
158                return new ArrayRealVector(obs);
159        }
160
161        private void updateEstimates(RealVector parameters) {
162                double[] c = parameters.toArray();
163                double[] s = Arrays.copyOfRange(c, 0, camParLength);
164                finalCamera = new Camera(s);
165
166                finalViews = new ViewTransform[M];
167                int start = s.length;
168                for (int i = 0; i < M; i++) {
169                        double[] w = Arrays.copyOfRange(c, start, start + viewParLength);
170                        finalViews[i] = new ViewTransform(w);
171                        start = start + w.length;
172                }
173        }
174
175        /**
176         * Returns the optimized camera parameters.
177         *
178         * @return the optimized camera parameters
179         */
180        Camera getFinalCamera() {
181                return finalCamera;
182        }
183
184        /**
185         * Returns the optimized view parameters.
186         *
187         * @return the optimized view parameters
188         */
189        ViewTransform[] getFinalViews() {
190                return finalViews;
191        }
192
193}