Commit ad2b3673 authored by Marek Horský's avatar Marek Horský
Browse files

Implements AvgFaceNNGPU visitor and exposes the option in UI.

parent 73ecc862

FaceEngines/src/main/java/cz/fidentis/analyst/engines/face/batch/registration/BatchFaceRegistrationServices.java

+7 −6
Original line number Diff line number Diff line
@@ -37,12 +37,7 @@ public class BatchFaceRegistrationServices { 
         * Uses Iterative Closets Point algorithm reimplemented for GPU

         * No landmarks are required.

         */

        ICP_GPU,



        /**

         * Purely for testing purposes, until a proper option in UI is implemented.

         */

        GPU_BATCH_REGISTRATION_PLACEHOLDER

        ICP_GPU

    }



    /**
@@ -63,6 +58,12 @@ public class BatchFaceRegistrationServices { 
         */

        NEAREST_NEIGHBOURS,



        /**

         * The average face metamorphose to other faces by searching nearest neighbors on GPU,

         * i.e., the closest vertices of registered faces.

         */

        NEAREST_NEIGHBOURS_GPU,



        /**

         * The average face metamorphose to other faces by projecting its vertices

         * to registered faces using ray-casting computed on CPU

FaceEngines/src/main/java/cz/fidentis/analyst/engines/face/batch/registration/impl/AverageFaceServices.java

0 → 100644
+95 −0
Original line number Diff line number Diff line 
package cz.fidentis.analyst.engines.face.batch.registration.impl;



import com.jogamp.opencl.CLContext;

import cz.fidentis.analyst.data.face.HumanFace;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshConfig;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshVisitor;

import cz.fidentis.analyst.engines.face.FaceStateServices;

import cz.fidentis.analyst.engines.face.batch.registration.BatchFaceRegistrationConfig;



import static cz.fidentis.analyst.engines.face.batch.registration.BatchFaceRegistrationServices.RegistrationStrategy.NONE;



/**

 * This class provides average face calculation methods for the batch registration process.

 *

 * @author Marek Horský

 */

class AverageFaceServices {



    /**

     * Update the average face geometry by taking into account given superimposed face.

     *

     * @param initFace         A template face. Its geometry is used to compute the average geometry. However,

     *                         the geometry of the {@code initFace} remains unchanged (a new mesh is created instead)

     * @param superimposedFace A face newly registered to the {@code initFace}. Its geometry is used to update the average face geometry

     * @param avgFaceVisitor   Average face visitor. If {@code null}, then new visitor is created.

     * @return Either newly created visitor or the {@code avgFaceVisitor}

     */

    static AvgMeshVisitor computeAvgFaceNN(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor, BatchFaceRegistrationConfig config) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateKdTree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getNearestNeighborsVisitor()

                : avgFaceVisitor;

        superimposedFace.getKdTree().accept(ret);



        return ret;

    }



    static AvgMeshVisitor computeAvgFaceNNGPU(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor, BatchFaceRegistrationConfig config, CLContext clContext) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateLeftBalancedKdTree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getNearestNeighborsGpuVisitor(clContext)

                : avgFaceVisitor;



        ret.visitKdTree(superimposedFace.getLeftBalancedKdTree()); // replace with accept after resolving conflict



        return ret;

    }



    /**

     * Update the average face geometry by taking into account given superimposed face.

     *

     * @param initFace         A template face. Its geometry is used to compute the average geometry. However,

     *                         the geometry of the {@code initFace} remains unchanged (a new mesh is created instead)

     * @param superimposedFace A face newly registered to the {@code initFace}. Its geometry is used to update the average face geometry

     * @param avgFaceVisitor   Average face visitor. If {@code null}, then new visitor is created.

     * @return Either newly created visitor or the {@code avgFaceVisitor}

     */

    static AvgMeshVisitor computeAvgFaceRT(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor, BatchFaceRegistrationConfig config) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateOctree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getRayCastingVisitor()

                : avgFaceVisitor;

        superimposedFace.getOctree().accept(ret);



        return ret;

    }



    static AvgMeshVisitor computeAvgFaceRTGPU(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor, BatchFaceRegistrationConfig config) {

        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), config.glContext())).getRayCastingGpuVisitor()

                : avgFaceVisitor;



        superimposedFace.getMeshModel().compute(ret);



        return ret;

    }

}

FaceEngines/src/main/java/cz/fidentis/analyst/engines/face/batch/registration/impl/BatchFaceRegistrationImpl.java

+13 −70
Original line number Diff line number Diff line 
package cz.fidentis.analyst.engines.face.batch.registration.impl;



import com.jogamp.opencl.CLContext;

import cz.fidentis.analyst.data.face.HumanFace;

import cz.fidentis.analyst.data.mesh.MeshModel;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshConfig;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshVisitor;

import cz.fidentis.analyst.engines.face.FaceRegistrationServices;

import cz.fidentis.analyst.engines.face.FaceRegistrationServicesOpenCL;
@@ -15,8 +15,6 @@ import cz.fidentis.analyst.opencl.OpenCLServices; 


import java.util.Objects;



import static cz.fidentis.analyst.engines.face.batch.registration.BatchFaceRegistrationServices.RegistrationStrategy.NONE;



/**

 * N:N registration and/or the computation of the average face.

 *
@@ -27,6 +25,7 @@ public class BatchFaceRegistrationImpl implements BatchFaceRegistration { 
    private final BatchFaceRegistrationConfig config;

    private final IcpConfig icpConfig;

    private final HumanFace templateFace;

    private final CLContext clContext;



    private FaceRegistrationServicesOpenCL faceRegistrationServicesOpenCL;

    private AvgMeshVisitor avgFaceVisitor = null;
@@ -43,6 +42,7 @@ public class BatchFaceRegistrationImpl implements BatchFaceRegistration { 
    public BatchFaceRegistrationImpl(HumanFace templateFace, BatchFaceRegistrationConfig config) {

        this.templateFace = Objects.requireNonNull(templateFace);

        this.config = config;

        this.clContext = OpenCLServices.createContext();



        switch (config.regStrategy()) {

            case ICP -> {
@@ -57,8 +57,8 @@ public class BatchFaceRegistrationImpl implements BatchFaceRegistration { 
                        sampling,

                        config.icpAutoCropSince());

            }

            case ICP_GPU, GPU_BATCH_REGISTRATION_PLACEHOLDER -> {

                this.faceRegistrationServicesOpenCL = new FaceRegistrationServicesOpenCL(OpenCLServices.createContext());

            case ICP_GPU -> {

                this.faceRegistrationServicesOpenCL = new FaceRegistrationServicesOpenCL(clContext);

                PointSamplingConfig sampling = (config.icpSubsampling() == 0)

                        ? new PointSamplingConfig(PointSamplingConfig.Method.NO_SAMPLING, config.icpSubsampling())

                        : new PointSamplingConfig(PointSamplingConfig.Method.RANDOM, config.icpSubsampling());
@@ -82,15 +82,16 @@ public class BatchFaceRegistrationImpl implements BatchFaceRegistration { 
            }

            case ICP -> FaceRegistrationServices.alignMeshes(face, icpConfig);

            case GPA -> FaceRegistrationServices.alignFeaturePoints(templateFace, face, config.scale());

            case ICP_GPU, GPU_BATCH_REGISTRATION_PLACEHOLDER -> faceRegistrationServicesOpenCL.alignMeshes(face, icpConfig);

            case ICP_GPU -> faceRegistrationServicesOpenCL.alignMeshes(face, icpConfig);

            default -> throw new IllegalStateException("Unexpected value: " + config.regStrategy());

        }



        avgFaceVisitor = switch (config.avgFaceStrategy()) {

            case NONE -> null;

            case NEAREST_NEIGHBOURS -> computeAvgFaceNN(templateFace, face, avgFaceVisitor);

            case PROJECTION_CPU -> computeAvgFaceRT(templateFace, face, avgFaceVisitor);

            case PROJECTION_GPU -> computeAvgFaceRTGPU(templateFace, face, avgFaceVisitor);

            case NEAREST_NEIGHBOURS -> AverageFaceServices.computeAvgFaceNN(templateFace, face, avgFaceVisitor, config);

            case NEAREST_NEIGHBOURS_GPU -> AverageFaceServices.computeAvgFaceNNGPU(templateFace, face, avgFaceVisitor, config, clContext);

            case PROJECTION_CPU -> AverageFaceServices.computeAvgFaceRT(templateFace, face, avgFaceVisitor, config);

            case PROJECTION_GPU -> AverageFaceServices.computeAvgFaceRTGPU(templateFace, face, avgFaceVisitor, config);

        };

    }
@@ -103,67 +104,9 @@ public class BatchFaceRegistrationImpl implements BatchFaceRegistration { 
    public void release() {

        if (faceRegistrationServicesOpenCL != null) {

            faceRegistrationServicesOpenCL.release();

            //TODO Release context as well?

        }

    }



    /**

     * Update the average face geometry by taking into account given superimposed face.

     *

     * @param initFace A template face. Its geometry is used to compute the average geometry. However,

     *                 the geometry of the {@code initFace} remains unchanged (a new mesh is created instead)

     * @param superimposedFace A face newly registered to the {@code initFace}. Its geometry is used to update the average face geometry

     * @param avgFaceVisitor Average face visitor. If {@code null}, then new visitor is created.

     * @return Either newly created visitor or the {@code avgFaceVisitor}

     */

    protected AvgMeshVisitor computeAvgFaceNN(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateKdTree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getNearestNeighborsVisitor()

                : avgFaceVisitor;

        superimposedFace.getKdTree().accept(ret);



        return ret;

        }



    /**

     * Update the average face geometry by taking into account given superimposed face.

     *

     * @param initFace A template face. Its geometry is used to compute the average geometry. However,

     *                 the geometry of the {@code initFace} remains unchanged (a new mesh is created instead)

     * @param superimposedFace A face newly registered to the {@code initFace}. Its geometry is used to update the average face geometry

     * @param avgFaceVisitor Average face visitor. If {@code null}, then new visitor is created.

     * @return Either newly created visitor or the {@code avgFaceVisitor}

     */

    protected AvgMeshVisitor computeAvgFaceRT(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateOctree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getRayCastingVisitor()

                : avgFaceVisitor;

        superimposedFace.getOctree().accept(ret);



        return ret;

        if (clContext != null) {

            clContext.release();

        }



    protected AvgMeshVisitor computeAvgFaceRTGPU(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), config.glContext())).getRayCastingGpuVisitor()

                : avgFaceVisitor;



        superimposedFace.getMeshModel().compute(ret);



        return ret;

    }

}

FaceEngines/src/main/java/cz/fidentis/analyst/engines/face/batch/registration/BatchFaceRegistrationGPUImpl.java→FaceEngines/src/main/java/cz/fidentis/analyst/engines/face/batch/registration/impl/BatchFaceRegistrationParallelImpl.java

+12 −53
Original line number Diff line number Diff line 
package cz.fidentis.analyst.engines.face.batch.registration;

package cz.fidentis.analyst.engines.face.batch.registration.impl;



import com.jogamp.opencl.CLContext;

import cz.fidentis.analyst.data.face.HumanFace;

import cz.fidentis.analyst.data.mesh.MeshModel;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshConfig;

import cz.fidentis.analyst.engines.avgmesh.AvgMeshVisitor;

import cz.fidentis.analyst.engines.face.FaceRegistrationServicesOpenCL;

import cz.fidentis.analyst.engines.face.FaceStateServices;

import cz.fidentis.analyst.engines.face.batch.registration.BatchFaceRegistrationConfig;

import cz.fidentis.analyst.engines.icp.IcpConfig;

import cz.fidentis.analyst.engines.sampling.PointSamplingConfig;

import cz.fidentis.analyst.opencl.memory.CLResources;
@@ -15,23 +15,23 @@ import java.util.List; 
import java.util.Queue;

import java.util.concurrent.atomic.AtomicBoolean;



import static cz.fidentis.analyst.engines.face.batch.registration.BatchFaceRegistrationServices.RegistrationStrategy.NONE;



/**

 * Provides batch face registration and face average functionality powered by GPU.

 *

 * @author Marek Horský

 */

public class BatchFaceRegistrationGPUImpl implements CLResources {

public class BatchFaceRegistrationParallelImpl implements CLResources {

    private final FaceRegistrationServicesOpenCL faceRegistrationServicesOpenCL;

    private final CLContext clContext;

    private HumanFace templateFace;

    private BatchFaceRegistrationConfig config;

    private AvgMeshVisitor avgFaceVisitor;



    private IcpConfig icpConfig;



    public BatchFaceRegistrationGPUImpl(CLContext context) {

        faceRegistrationServicesOpenCL = new FaceRegistrationServicesOpenCL(context);

    public BatchFaceRegistrationParallelImpl(CLContext clContext) {

        this.clContext = clContext;

        this.faceRegistrationServicesOpenCL = new FaceRegistrationServicesOpenCL(clContext);

    }



    /**
@@ -72,7 +72,7 @@ public class BatchFaceRegistrationGPUImpl implements CLResources { 
     *

     * @param faceBatch face batch

     */

    public void register(AtomicBoolean isRunning, Queue<HumanFace> faceBatch) {

    public void registerAsync(AtomicBoolean isRunning, Queue<HumanFace> faceBatch) {

        while (isRunning.get()) {

            HumanFace face = faceBatch.poll();

            if (face != null) {
@@ -91,9 +91,10 @@ public class BatchFaceRegistrationGPUImpl implements CLResources { 
        for (HumanFace face : faceBatch) {

            avgFaceVisitor = switch (config.avgFaceStrategy()) {

                case NONE -> null;

                case NEAREST_NEIGHBOURS -> computeAvgFaceNN(templateFace, face, avgFaceVisitor);

                case PROJECTION_CPU -> computeAvgFaceRT(templateFace, face, avgFaceVisitor);

                case PROJECTION_GPU -> computeAvgFaceRTGPU(templateFace, face, avgFaceVisitor);

                case NEAREST_NEIGHBOURS -> AverageFaceServices.computeAvgFaceNN(templateFace, face, avgFaceVisitor, config);

                case NEAREST_NEIGHBOURS_GPU -> AverageFaceServices.computeAvgFaceNNGPU(templateFace, face, avgFaceVisitor, config, clContext);

                case PROJECTION_CPU -> AverageFaceServices.computeAvgFaceRT(templateFace, face, avgFaceVisitor, config);

                case PROJECTION_GPU -> AverageFaceServices.computeAvgFaceRTGPU(templateFace, face, avgFaceVisitor, config);

            };

        }
@@ -105,46 +106,4 @@ public class BatchFaceRegistrationGPUImpl implements CLResources { 
    public void release() {

        faceRegistrationServicesOpenCL.release();

    }



    protected AvgMeshVisitor computeAvgFaceNN(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateKdTree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getNearestNeighborsVisitor()

                : avgFaceVisitor;

        superimposedFace.getKdTree().accept(ret);



        return ret;

    }



    protected AvgMeshVisitor computeAvgFaceRT(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        // If the face was moved by registration, then the spatial ordering structure was removed => create it

        // If no transformation was made, then you can use old structure, if exists

        FaceStateServices.updateOctree(

                superimposedFace,

                config.regStrategy() == NONE ? FaceStateServices.Mode.COMPUTE_IF_ABSENT : FaceStateServices.Mode.COMPUTE_ALWAYS

        );



        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), null)).getRayCastingVisitor()

                : avgFaceVisitor;

        superimposedFace.getOctree().accept(ret);



        return ret;

    }



    protected AvgMeshVisitor computeAvgFaceRTGPU(HumanFace initFace, HumanFace superimposedFace, AvgMeshVisitor avgFaceVisitor) {

        AvgMeshVisitor ret = (avgFaceVisitor == null)

                ? (new AvgMeshConfig(initFace.getMeshModel(), config.glContext())).getRayCastingGpuVisitor()

                : avgFaceVisitor;



        superimposedFace.getMeshModel().compute(ret);



        return ret;

    }

}

FaceEngines/src/test/java/cz/fidentis/analyst/engines/face/FaceTester.java

+9 −9
Original line number Diff line number Diff line
@@ -35,14 +35,14 @@ public class FaceTester { 
            }

        }

        return checkLandmarksForEquality(face1.getLandmarks(), face2.getLandmarks())

                ;

                //&& checkSymmetryPlanes(face1.getSymmetryPlane(), face2.getSymmetryPlane());

                && checkSymmetryPlanes(face1.getSymmetryPlane(), face2.getSymmetryPlane());

    }



    static boolean checkSymmetryPlanes(Plane symmetryPlane1, Plane symmetryPlane2) {

        if(symmetryPlane1 == null || symmetryPlane2 == null) return false;

        return symmetryPlane1.getPlanePoint().epsilonEquals(symmetryPlane2.getPlanePoint(), EPS)

                && symmetryPlane1.getNormal().epsilonEquals(symmetryPlane2.getNormal(), EPS);

    static boolean checkSymmetryPlanes(Plane plane1, Plane plane2) {

        if (plane2 == plane1) return true;

        if (plane1 == null || plane2 == null) return false;

        return plane1.getPlanePoint().epsilonEquals(plane2.getPlanePoint(), EPS)

                && plane1.getNormal().epsilonEquals(plane2.getNormal(), EPS);

    }



    static boolean checkLandmarksForEquality(Landmarks landmarks1, Landmarks landmarks2) {