Commit 796d7224 authored by Jakub Kolman's avatar Jakub Kolman
Browse files

Merge branch 'issue-123/procrustes-visitor'

parents 5b39bd6c 4207346e
Loading
Loading
Loading
Loading
+158 −112
Original line number Diff line number Diff line
package cz.fidentis.analyst.face;

import cz.fidentis.analyst.Logger;
import cz.fidentis.analyst.feature.FeaturePoint;
import cz.fidentis.analyst.icp.IcpTransformer;
import cz.fidentis.analyst.icp.Quaternion;
import cz.fidentis.analyst.procrustes.ProcrustesAnalysis;
import cz.fidentis.analyst.symmetry.Plane;
import cz.fidentis.analyst.visitors.mesh.sampling.PointSampling;
import java.util.zip.DataFormatException;
import cz.fidentis.analyst.visitors.procrustes.ProcrustesVisitor;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import javax.vecmath.Matrix3d;
import javax.vecmath.Matrix4d;
import javax.vecmath.Point3d;
@@ -24,8 +26,8 @@ import javax.vecmath.Vector3d;
public class HumanFaceUtils {

    /**
     * Superimpose two faces by applying iterative closest points (ICP) algorithm
     * on their triangular meshes.
     * Superimpose two faces by applying iterative closest points (ICP)
     * algorithm on their triangular meshes.
     *
     * @param staticFace A face that remains unchanged.
     * @param transformedFace A face to be transformed.
@@ -34,9 +36,11 @@ public class HumanFaceUtils {
     * @param scale Whether to scale face as well
     * @param error Acceptable error (a number bigger than or equal to zero).
     * @param samplingStrategy Downsampling strategy
     * @param recomputeKdTree If {@code true} and the k-d tree of the {@code transformedFace} exists, 
     *        the it automatically re-computed. Otherwise, it is simply removed.
     * @return ICP visitor that holds the transformations performed on the {@code transformedFace}.
     * @param recomputeKdTree If {@code true} and the k-d tree of the
     * {@code transformedFace} exists, the it automatically re-computed.
     * Otherwise, it is simply removed.
     * @return ICP visitor that holds the transformations performed on the
     * {@code transformedFace}.
     */
    public static IcpTransformer alignMeshes(
            HumanFace staticFace, HumanFace transformedFace,
@@ -102,11 +106,14 @@ public class HumanFaceUtils {
     * Transform the face.
     *
     * @param face Face to be transformed.
     * @param rotation Rotation vector denoting the rotation angle around axes X, Y, and Z.
     * @param translation Translation vector denoting the translation in the X, Y, and Z direction.
     * @param rotation Rotation vector denoting the rotation angle around axes
     * X, Y, and Z.
     * @param translation Translation vector denoting the translation in the X,
     * Y, and Z direction.
     * @param scale Scale factor (1 = no scale).
     * @param recomputeKdTree If {@code true} and the k-d tree of the {@code transformedFace} exists, 
     *        the it automatically re-computed. Otherwise, it is simply removed.
     * @param recomputeKdTree If {@code true} and the k-d tree of the
     * {@code transformedFace} exists, the it automatically re-computed.
     * Otherwise, it is simply removed.
     */
    public static void transformFace(HumanFace face, Vector3d rotation, Vector3d translation, double scale, boolean recomputeKdTree) {
        Quaternion rot = new Quaternion(rotation.x, rotation.y, rotation.z, 1.0);
@@ -120,7 +127,6 @@ public class HumanFaceUtils {
                    transformNormal(meshPoint.getNormal(), rot);
                });

        
        // update k-d of transformed face:
        if (face.hasKdTree()) {
            if (recomputeKdTree) {
@@ -147,13 +153,16 @@ public class HumanFaceUtils {
    }

    /**
     * Transforms one face so that its symmetry plane fits the symmetry plane of the other face.
     * Transforms one face so that its symmetry plane fits the symmetry plane of
     * the other face.
     *
     * @param staticFace a human face that remains unchanged
     * @param transformedFace a human face that will be transformed
     * @param recomputeKdTree  If {@code true} and the k-d tree of the {@code transformedFace} exists, 
     *        the it automatically re-computed. Otherwise, it is simply removed.
     * @param preserveUpDir If {@code false}, then the object can be rotated around the target's normal arbitrarily.
     * @param recomputeKdTree If {@code true} and the k-d tree of the
     * {@code transformedFace} exists, the it automatically re-computed.
     * Otherwise, it is simply removed.
     * @param preserveUpDir If {@code false}, then the object can be rotated
     * around the target's normal arbitrarily.
     */
    public static void alignSymmetryPlanes(
            HumanFace staticFace, HumanFace transformedFace,
@@ -212,36 +221,27 @@ public class HumanFaceUtils {
    }

    /**
     * Superimpose two faces by applying Procrustes on their feature points.
     * Be aware that both faces are transformed into the origin of the coordinate system!
     * Superimpose two faces by applying Procrustes on their feature points. Be
     * aware that both faces are transformed into the origin of the coordinate
     * system!
     *
     * @param firstFace a human face that remains unchanged
     * @param secondFace a human face that will be transformed
     * @param scale Whether to scale faces as well
     * @param recomputeKdTree If {@code true} and the k-d tree of the {@code transformedFace} exists, 
     *        the it automatically re-computed. Otherwise, it is simply removed.
     * @param recomputeKdTree If {@code true} and the k-d tree of the
     * {@code transformedFace} exists, the it automatically re-computed.
     * Otherwise, it is simply removed.
     */
    public static void alignFeaturePoints(
    public static ProcrustesVisitor alignFeaturePoints(
            HumanFace firstFace, HumanFace secondFace,
            boolean scale, boolean recomputeKdTree) {

        ProcrustesAnalysis pa = null;
        try {
            pa = new ProcrustesAnalysis(firstFace, secondFace, scale);
            pa.analyze();
        } catch (DataFormatException e) {
            Logger.print("Procrustes Analysis experienced exception");
            return;
        }
        // get transformation values
        ProcrustesVisitor pv = new ProcrustesVisitor(firstFace, secondFace, scale);

        // superimpose face towards the static face
        secondFace.getMeshModel().compute(pv, false);

        // update k-d of transformed faces:
        if (firstFace.hasKdTree()) { 
            if (recomputeKdTree) {
                firstFace.computeKdTree(true);
            } else {
                firstFace.removeKdTree();
            }
        }
        // update k-d of transformed faces:
        if (secondFace.hasKdTree()) {
            if (recomputeKdTree) {
@@ -252,24 +252,34 @@ public class HumanFaceUtils {
        }

        // update bounding box, which is always present:
        firstFace.updateBoundingBox();
        secondFace.updateBoundingBox();

        // transform feature points:
        if (firstFace.hasFeaturePoints()) {
            // TODO
        }
        if (secondFace.hasFeaturePoints()) {
            // TODO
        }
        // better option would be use of secondFace.setFeaturePoints(), but it doesnt repaint them to their new locations
        if (secondFace.hasFeaturePoints() && pv.getTransformation().getFaceModel().getFeaturePointsMap() != null) {
            moveFeaturePoints(secondFace, pv.getTransformation().getFaceModel().getFeaturePointsMap());

        // transform symmetry plane:
        if (firstFace.hasSymmetryPlane()) {
            // TODO
//            secondFace.setFeaturePoints(
//                    new ArrayList<>(pv.getTransformation().getFaceModel().getFeaturePointsMap().values()));
        }
        // transform symmetry plane:
        if (secondFace.hasSymmetryPlane()) {
            // TODO
            Vector3d adjustment = pv.getTransformation().getCentroidAdjustment();
            double transformationScaleValue = pv.getTransformation().getScale();
            Quaternion rotation = pv.getTransformation().getMatrixAsQuaternion(
                    pv.getTransformation().getRotationMatrix());
            secondFace.getFeaturePoints().parallelStream().forEach(fp -> {
                secondFace.setSymmetryPlane(transformPlane(
                        secondFace.getSymmetryPlane(), rotation, adjustment, transformationScaleValue));
            });
            secondFace.getFeaturePoints().parallelStream().forEach(fp -> {
                secondFace.setSymmetryPlane(transformPlane(
                        secondFace.getSymmetryPlane(),
                        pv.getTransformation().getMatrixAsQuaternion(pv.getTransformation().getIdentityMatrix()),
                        pv.getTransformation().getSuperImpositionAdjustment(), 1));
            });
        }
        return pv;
    }

    /**
@@ -354,7 +364,43 @@ public class HumanFaceUtils {
        return new Plane(retPlane.getNormal(), dist);
    }

    /**
     * Moves feature points to calculated new position by computing procrustes
     * analysis. {@link cz.fidentis.analyst.procrustes.ProcrustesAnalysis}. If
     * feature point was not used in the analysis, it's position will be kept as
     * it had in the original list.
     *
     * Move of the feature point is done to remove them from user vision first.
     *
     * @param secondFace
     * @param movedFeaturePoints
     */
    private static void moveFeaturePoints(HumanFace secondFace, HashMap<Integer, FeaturePoint> movedFeaturePoints) {
        secondFace.getFeaturePoints().forEach(fp -> {
            FeaturePoint movedFp = movedFeaturePoints.get(fp.getFeaturePointType().getType());
            if (movedFp != null) {
                if (fp.getFeaturePointType().getType()
                        != movedFp.getFeaturePointType().getType()) {
                    throw new RuntimeException("Types do not correspond");
                }
                fp.getPosition().x = movedFp.getX();
                fp.getPosition().y = movedFp.getY();
                fp.getPosition().z = movedFp.getZ();
            }
        });
    }

    
    /**
     * Sorts List by featurePoint type property
     *
     * @param featurePointList
     * @return ordered list of feature points by type
     */
    protected static List<FeaturePoint> sortListByFeaturePointType(List<FeaturePoint> featurePointList) {
        Collections.sort(
                featurePointList, Comparator.comparingInt(fp -> fp.getFeaturePointType().getType())
        );
        return featurePointList;
    }

}
+244 −102

File changed.

Preview size limit exceeded, changes collapsed.

+155 −11
Original line number Diff line number Diff line
@@ -20,11 +20,12 @@ public class ProcrustesAnalysisFaceModel {
    private final HumanFace humanFace;
    private HashMap<Integer, FeaturePoint> featurePointsMap;        // sorted by feature point type
    private List<MeshPoint> vertices;
    private MeshModel meshModel;
    private final MeshModel meshModel;
    private final HashMap<Integer, Integer> featurePointTypeCorrespondence;

    /**
     * constructor creating inner attributes used in procrustes analysis from face
     * constructor creating inner attributes used in procrustes analysis from
     * face
     *
     * @param face
     */
@@ -44,13 +45,69 @@ public class ProcrustesAnalysisFaceModel {
    }

    /**
     * sets feature points map and also sets feature point on human face for visualisation
     * Constructor for face model if there is a need for a subset selection of
     * feature points before applying procrustes analysis.
     *
     * @param face
     * @param viablePoints
     */
    public ProcrustesAnalysisFaceModel(HumanFace face, List<Integer> viablePoints) {
        this.humanFace = face;
        // getFeaturePoints() returns unmodifiable List. To sort it we need to make copy first
        List<FeaturePoint> modifiableFeaturePointList = getSelectionOfFeaturePoints(face.getFeaturePoints(), viablePoints);

        // Currently set feature points is not working as inteded because gui doesn't repaint them after changing values
//        face.setFeaturePoints(modifiableFeaturePointList);

        // To be able to move vertices we have to make a copy of them because  getFacets() returns unmodifiable List
        List<MeshPoint> modifiableVertices = new ArrayList<>(face.getMeshModel().getFacets().get(0).getVertices());

        this.vertices = modifiableVertices;
        this.meshModel = face.getMeshModel();
        this.featurePointsMap = createFeaturePointMap(
                sortListByFeaturePointType(modifiableFeaturePointList));
        this.featurePointTypeCorrespondence = createFeaturePointTypeCorrespondence(face.getFeaturePoints());
    }

    /**
     * Use this constructor instead of the
     * {@link #ProcrustesAnalysisFaceModel(cz.fidentis.analyst.face.HumanFace, java.util.List)}
     * until the
     * {@link cz.fidentis.analyst.face.HumanFace#setFeaturePoints(java.util.List)}
     * is fixed and fires event to render in GUI a new set of feature points.
     *
     * Feature points that are not used in subset are moved to position of
     * modelCentroid.
     *
     * @param humanFace1
     * @param viablePoints
     * @param modelCentroid1
     */
//    ProcrustesAnalysisFaceModel(HumanFace face, ArrayList<Integer> viablePoints, Point3d modelCentroid) {
//        this.humanFace = face;
//        // getFeaturePoints() returns unmodifiable List. To sort it we need to make copy first
//        List<FeaturePoint> modifiableFeaturePointList = getSelectionOfFeaturePoints(face.getFeaturePoints(), viablePoints);
//
//        // To be able to move vertices we have to make a copy of them because  getFacets() returns unmodifiable List
//        List<MeshPoint> modifiableVertices = new ArrayList<>(face.getMeshModel().getFacets().get(0).getVertices());
//
//        this.vertices = modifiableVertices;
//        this.meshModel = face.getMeshModel();
//        this.featurePointsMap = createFeaturePointMap(
//                sortListByFeaturePointType(modifiableFeaturePointList));
//        this.featurePointTypeCorrespondence = createFeaturePointTypeCorrespondence(face.getFeaturePoints());
//    }

    /**
     * Sets feature points map and also sets feature point on human face for
     * visualisation.
     *
     * @param featurePointsMap
     */
    public void setFeaturePointsMap(HashMap<Integer, FeaturePoint> featurePointsMap) {
        this.featurePointsMap = featurePointsMap;
        this.humanFace.setFeaturePoints(getFeaturePointValues());
        readjustFeaturePoints(featurePointsMap);
//        this.humanFace.setFeaturePoints(getFeaturePointValues());
    }

    public MeshModel getMeshModel() {
@@ -78,9 +135,53 @@ public class ProcrustesAnalysisFaceModel {
    }

    /**
     * Creates corresponding key value pair of matrix row index and a feature point type value.
     * It is used so we can get back type of feature point after working with matrices where the type is not stored,
     * but instead are used matrix indexes.
     * Creates a subset of selected feature points that can be used for
     * procrustes analysis. Each of the feature point types contained in the
     * subset is contained in both faces.
     *
     * @param featurePoints
     * @param viablePoints
     * @return
     */
    private List<FeaturePoint> getSelectionOfFeaturePoints(List<FeaturePoint> featurePoints, List<Integer> viablePoints) {
        ArrayList<FeaturePoint> selection = new ArrayList<>();
        featurePoints.forEach(fp -> {
            if (viablePoints.contains(fp.getFeaturePointType().getType())) {
                selection.add(fp);
            }
        });
        return selection;
    }

    /**
     * {@link #getSelectionOfFeaturePoints(java.util.List, java.util.List)}
     * Feature points that are not in subset are moved to position of
     * modelCentroid.
     *
     * @param featurePoints
     * @param viablePoints
     * @param modelCentroid
     * @return
     */
//    private List<FeaturePoint> getSelectionOfFeaturePoints(List<FeaturePoint> featurePoints, ArrayList<Integer> viablePoints, Point3d modelCentroid) {
//        ArrayList<FeaturePoint> selection = new ArrayList<>();
//        featurePoints.forEach(fp -> {
//            if (viablePoints.contains(fp.getFeaturePointType().getType())) {
//                selection.add(fp);
//            } else {
//                fp.getPosition().x = modelCentroid.x;
//                fp.getPosition().y = modelCentroid.y;
//                fp.getPosition().z = modelCentroid.z;
//            }
//        });
//        return selection;
//    }

    /**
     * Creates corresponding key value pair of matrix row index and a feature
     * point type value. It is used so we can get back type of feature point
     * after working with matrices where the type is not stored, but instead are
     * used matrix indexes.
     *
     * @param fpList
     * @return
@@ -109,8 +210,7 @@ public class ProcrustesAnalysisFaceModel {
    /**
     * Creates feature point hash map for internal use in Procrustes Analysis.
     *
     * Map key is feature point type (Integer).
     * Value is feature point itself.
     * Map key is feature point type (Integer). Value is feature point itself.
     *
     * @param featurePointList
     * @return
@@ -123,4 +223,48 @@ public class ProcrustesAnalysisFaceModel {
        return map;
    }

    private void readjustFeaturePoints(HashMap<Integer, FeaturePoint> featurePointsMap) {
//        if (featurePointsMap.size() != this.humanFace.getFeaturePoints().size()) {
//            readjustSelection(featurePointsMap);
//        } else {
        this.humanFace.getFeaturePoints().forEach(fp -> {
            FeaturePoint movedFp = featurePointsMap.get(fp.getFeaturePointType().getType());
            if (movedFp != null) {
                if (fp.getFeaturePointType().getType()
                        != movedFp.getFeaturePointType().getType()) {
                    throw new RuntimeException("Types do not correspond");
                }
                fp.getPosition().x = movedFp.getX();
                fp.getPosition().y = movedFp.getY();
                fp.getPosition().z = movedFp.getZ();
            }
        });
//    }
    }

//    /**
//     * Moves subset of feature points to their new position. If point doesn't
//     * belong to subset it is moved to (0,0,0) instead.
//     *
//     * @param featurePointsMap
//     */
//    private void readjustSelection(HashMap<Integer, FeaturePoint> featurePointsMap) {
//        this.humanFace.getFeaturePoints().forEach(fp -> {
//            FeaturePoint movedFp = featurePointsMap.get(fp.getFeaturePointType().getType());
//            if (movedFp != null) {
//                if (fp.getFeaturePointType().getType()
//                        != movedFp.getFeaturePointType().getType()) {
//                    throw new RuntimeException("Types do not correspond");
//                }
//                fp.getPosition().x = movedFp.getX();
//                fp.getPosition().y = movedFp.getY();
//                fp.getPosition().z = movedFp.getZ();
//            } else {
//                fp.getPosition().x = 0;
//                fp.getPosition().y = 0;
//                fp.getPosition().z = 0;
//            }
//        });
//    }

}
+157 −0
Original line number Diff line number Diff line
package cz.fidentis.analyst.procrustes;

import cz.fidentis.analyst.icp.Quaternion;
import javax.vecmath.Vector3d;
import org.ejml.simple.SimpleMatrix;

/**
 * Class used to store information about transformation that needs to be applied
 * to a second face when computing procrustes analysis.
 *
 * @author Jakub Kolman
 */
public class ProcrustesTransformation {

    private double scale;
    // used for adjustment to move feature points (0,0,0) so the face is rotated around the centroid point
    private Vector3d centroidAdjustment;
    // used to move face to superimposed poisition over the original human face 
    private Vector3d superImpositionAdjustment;
    private SimpleMatrix rotationMatrix;
    private ProcrustesAnalysisFaceModel faceModel;

    /**
     * constructor
     *
     * @param scale determines by how much should be faced enlarged or minimized
     * @param adjustment determines how should be face moved to make faces
     * superimposed considering feature points
     * @param rotationMatrix determines how to rotate face
     * @param updatedFaceModel holds information about adjusted face
     */
    public ProcrustesTransformation(
            double scale,
            Vector3d centroidAdjustment,
            Vector3d superImpositionAdjustment,
            SimpleMatrix rotationMatrix,
            ProcrustesAnalysisFaceModel updatedFaceModel) {
        this.scale = scale;
        this.centroidAdjustment = centroidAdjustment;                   
        this.superImpositionAdjustment = superImpositionAdjustment;
        this.rotationMatrix = rotationMatrix;
        this.faceModel = updatedFaceModel;
    }

    /**
     * Constructor (for more info take a look on 
     * {@link #ProcrustesTransformation(double, javax.vecmath.Vector3d, org.ejml.simple.SimpleMatrix, cz.fidentis.analyst.procrustes.ProcrustesAnalysisFaceModel) }
     *
     * @param adjustment
     * @param rotationMatrix
     */
    public ProcrustesTransformation(Vector3d centroidAdjustment, Vector3d superImpositionAdjustment, SimpleMatrix rotationMatrix) {
        this.scale = 1;
        this.centroidAdjustment = centroidAdjustment;
        this.superImpositionAdjustment = superImpositionAdjustment;
        this.rotationMatrix = rotationMatrix;
    }

    /**
     * Empty constructor with values that won't transform face. To set values later attribute setters can be used.
     * <pre>
     * rotation matrix is set to identity matrix
     * scale is 1
     * adjustment is none
     * </pre>
     * (for more info take a look on 
     * {@link #ProcrustesTransformation(double, javax.vecmath.Vector3d, org.ejml.simple.SimpleMatrix, cz.fidentis.analyst.procrustes.ProcrustesAnalysisFaceModel) }
     */
    public ProcrustesTransformation() {
        this.scale = 1;
        this.centroidAdjustment = new Vector3d(0,0,0);
        this.superImpositionAdjustment = new Vector3d(0,0,0);
        SimpleMatrix r = new SimpleMatrix(3, 3);    // identity matrix
        r.set(0, 0, 1);
        r.set(0, 1, 0);
        r.set(0, 2, 0);
        r.set(1, 0, 0);
        r.set(1, 1, 1);
        r.set(1, 2, 0);
        r.set(2, 0, 0);
        r.set(2, 1, 0);
        r.set(2, 2, 1);
        this.rotationMatrix = r;
    }

    public double getScale() {
        return this.scale;
    }

    public Vector3d getCentroidAdjustment() {
        return this.centroidAdjustment;
    }

    public Vector3d getSuperImpositionAdjustment() {
        return this.superImpositionAdjustment;
    }

    public SimpleMatrix getRotationMatrix() {
        return this.rotationMatrix;
    }

    public ProcrustesAnalysisFaceModel getFaceModel() {
        return faceModel;
    }

    public void setScale(double scale) {
        this.scale = scale;
    }

    public void setFaceModel(ProcrustesAnalysisFaceModel faceModel) {
        this.faceModel = faceModel;
    }

    /**
     * Calculates quaternion from rotation matrix.
     *
     * @return quaternion with rotation values
     */
    public Quaternion getMatrixAsQuaternion(SimpleMatrix matrix) {
        if (matrix != null) {           
            double w = Math.sqrt(1.0 + matrix.get(0, 0) + matrix.get(1, 1) + matrix.get(2, 2)) / 2.0;
            double w4 = (4.0 * w);
            double x = (matrix.get(2, 1) - matrix.get(1, 2)) / w4;
            double y = (matrix.get(0, 2) - matrix.get(2, 0)) / w4;
            double z = (matrix.get(1, 0) - matrix.get(0, 1)) / w4;
            return new Quaternion(x, y, z, w);
        } else {
            return null;
        }
    }

    public void setCentroidAdjustment(Vector3d centroidAdjustment) {
        this.centroidAdjustment = centroidAdjustment;
    }

    public void setSuperImpositionAdjustment(Vector3d superImpositionAdjustment) {
        this.superImpositionAdjustment = superImpositionAdjustment;
    }

    public void setRotationMatrix(SimpleMatrix rotationMatrix) {
        this.rotationMatrix = rotationMatrix;
    }
    
   public SimpleMatrix getIdentityMatrix() {
        SimpleMatrix r = new SimpleMatrix(3, 3);    // identity matrix
        r.set(0, 0, 1);
        r.set(0, 1, 0);
        r.set(0, 2, 0);
        r.set(1, 0, 0);
        r.set(1, 1, 1);
        r.set(1, 2, 0);
        r.set(2, 0, 0);
        r.set(2, 1, 0);
        r.set(2, 2, 1);
        return r;
   }
}
+129 −0

File added.

Preview size limit exceeded, changes collapsed.

Loading