Commit ea49e9c1 authored by xnovak8's avatar xnovak8
Browse files

* the ID-object storage now enables pivot-filtering

parent 6c371de1

src/main/java/pppcodes/algorithms/PPPCodeAlgorithm.java

+1 −1
Original line number Diff line number Diff line
@@ -154,7 +154,7 @@ public class PPPCodeAlgorithm extends MultipleOverlaysAlgorithm { 
        if (idObjectStorage != null) {

            for (LocalAbstractObject obj : insertedObjects) {

                try {

                    idObjectStorage.storeObject(obj);

                    idObjectStorage.storeObject(idObjectStorage.preprocessObject(obj));

                } catch (BucketStorageException ex) {

                    MetricIndexes.logger.severe("id-object storage failed: " + ex.getLocalizedMessage());

                    return false;

src/main/java/pppcodes/ids/IDObjectRAStorage.java

+6 −0
Original line number Diff line number Diff line
@@ -43,4 +43,10 @@ public interface IDObjectRAStorage<T extends LocalAbstractObject> { 
     */

    public Search<T> getAllObjects();

    

    /**

     * Preprocess the object before inserting it to the storage.

     * @param object object to be preprocessed 

     * @return the same preprocessed object

     */

    public T preprocessObject(T object);

}

src/main/java/pppcodes/ids/PPPDiskIndex.java

+31 −7
Original line number Diff line number Diff line
@@ -21,7 +21,10 @@ import messif.buckets.BucketStorageException; 
import messif.buckets.index.Search;

import messif.buckets.storage.impl.DiskStorage;

import messif.objects.LocalAbstractObject;

import messif.objects.PrecomputedDistancesFixedArrayFilter;

import messif.utility.Convert;

import mindex.MIndexPPCalculator;

import mindex.MIndexProperties;

import mindex.MetricIndex;

import mindex.MetricIndexes;
@@ -37,18 +40,23 @@ public class PPPDiskIndex extends SimpleLocatorIntegerTranslator implements IDOb 
    /** Directory into which this index should be serialized */

    private final File serializationDir;



    /** The set of pivots for precomputing distances (can be null) */

    protected LocalAbstractObject[] pivots;

    

    /** Creates the PPP disk index given a disk storage. */

    protected PPPDiskIndex(DiskStorage<LocalAbstractObject> storage, File serializationDir) {

    protected PPPDiskIndex(DiskStorage<LocalAbstractObject> storage, File serializationDir, LocalAbstractObject[] pivots) {

        super();

        this.intObjectIndex = new DiskStorageMemoryIntIndex(storage);

        this.serializationDir = serializationDir;

        this.pivots = pivots;

    }



    /** Creates the PPP disk index given a disk storage. */

    protected PPPDiskIndex(DiskStorageMemoryIntIndex intObjectIndex, File serializationDir, AtomicInteger nextID) {

    /** Creates the PPP disk index from an existing index and having ID of the next inserted object. */

    protected PPPDiskIndex(DiskStorageMemoryIntIndex intObjectIndex, File serializationDir, LocalAbstractObject[] pivots, AtomicInteger nextID) {

        super(nextID);

        this.intObjectIndex = intObjectIndex;

        this.serializationDir = serializationDir;

        this.pivots = pivots;

    }
@@ -63,12 +71,12 @@ public class PPPDiskIndex extends SimpleLocatorIntegerTranslator implements IDOb 
    /** Default properties prefix for configuration of the {@link #intObjectIndex}. */

    public static final String CONFIG_PREFIX = "storage.";

    

    public static PPPDiskIndex restoreFromDir(File dir) throws IOException {

    public static PPPDiskIndex restoreFromDir(File dir, LocalAbstractObject[] pivots) throws IOException {

        File file = new File(dir, SERIALIZATION_FILE);

        try (ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)))) {

            DiskStorageMemoryIntIndex index = (DiskStorageMemoryIntIndex) in.readObject();

            AtomicInteger nextID = new AtomicInteger(index.getMaxKey() + 1);

            return new PPPDiskIndex(index, dir, nextID);

            return new PPPDiskIndex(index, dir, pivots, nextID);

        } catch (NullPointerException |ClassNotFoundException | ClassCastException ex) {

            throw new IOException(ex);

        }
@@ -85,12 +93,15 @@ public class PPPDiskIndex extends SimpleLocatorIntegerTranslator implements IDOb 
        File storageFile = Convert.getParameterValue(storageProperties, "file", File.class, new File(dir, STORAGE_FILE));

        storageProperties.put("file", storageFile);



        // create pivots to calculate the precoputed distances

        LocalAbstractObject [] pivots = MIndexPPCalculator.initPivots(new MIndexProperties(configuration, "pivots"));

        

        // if the specified directory contains serialized file and data storage then restore it

        if (storageFile.exists() && new File(dir, SERIALIZATION_FILE).exists()) {

            return restoreFromDir(dir);

            return restoreFromDir(dir, pivots);

        }

        try {

            return new PPPDiskIndex(DiskStorage.create(LocalAbstractObject.class, storageProperties), dir);

            return new PPPDiskIndex(DiskStorage.create(LocalAbstractObject.class, storageProperties), dir, pivots);

        } catch (InstantiationException ex) {

            throw new IOException(ex);

        }
@@ -130,6 +141,19 @@ public class PPPDiskIndex extends SimpleLocatorIntegerTranslator implements IDOb 
        return intObjectIndex.getAllObjects();

    }    



    @Override

    public LocalAbstractObject preprocessObject(LocalAbstractObject object) {

        if (pivots != null) {

            float[] distances = new float[pivots.length];

            for (int distanceIndex = 0; distanceIndex < pivots.length; distanceIndex++) {

                distances[distanceIndex] = object.getDistance(pivots[distanceIndex]);

            }

            PrecomputedDistancesFixedArrayFilter filter = new PrecomputedDistancesFixedArrayFilter(object);

            filter.setFixedPivotsPrecompDist(distances);

        }

        return object;

    }        

    

    @Override

    public String getStringLocator(int id) {

        LocalAbstractObject object = readObject(id);

src/main/java/pppcodes/index/PPPCodeLeafCell.java

+7 −18
Original line number Diff line number Diff line
@@ -224,7 +224,7 @@ public class PPPCodeLeafCell extends VoronoiLeafCell<PPPCodeObject> implements M 
    @Override

    public synchronized SplitConfiguration insertObjects(List<PPPCodeObject> objects) throws AlgorithmMethodException {

        if (! ensureWriteOf(objects, getLevel())) {

            return new SplitConfiguration(false, 0, objects);

            return new SplitConfiguration(false, 0, objects, null);

        }



        PPPCodeReadWriter serializer = getPPPCodeRW();
@@ -237,8 +237,9 @@ public class PPPCodeLeafCell extends VoronoiLeafCell<PPPCodeObject> implements M 
                iterator.next().writeToBytes(writeBufferBits, serializer, pppSufixLength);

            } catch (BufferOverflowException ex) {

                objectCount += insertedObjs;

                //objectCount.addAndGet(insertedObjs);

                Logger.getLogger(PPPCodeLeafCell.class.getName()).log(Level.SEVERE, ex.getLocalizedMessage());

                return new SplitConfiguration(false, insertedObjs, objects.subList(insertedObjs, objects.size()));

                return new SplitConfiguration(false, insertedObjs, objects.subList(insertedObjs, objects.size()), null);

            }

            insertedObjs ++;

        }
@@ -376,21 +377,6 @@ public class PPPCodeLeafCell extends VoronoiLeafCell<PPPCodeObject> implements M 
        };

    }

        

    

//    @Override

//    public boolean containsGivenPivot(int pivotNumber) {

//        int thisLevel = getLevel();

//        if (thisLevel >= mIndex.getMaxLevel()) {

//            return false;

//        }

//        boolean [] indexes = (indexesOnNextLevel == null) ? null : indexesOnNextLevel.get();

//        if (indexes == null) {

//            indexes = getPPPCodeRW().readNextLevelPivots(getPPPCodeRW().createNewBuffer(data, getOccupationInBits()), mIndex.getMaxLevel() - thisLevel);

//            indexesOnNextLevel = new WeakReference<>(indexes);

//        }

//        return indexes[pivotNumber];

//    }    

//

    @Override

    public float reviseMinDistance(PartialQueryPPPDistanceCalculator calculator, float origDistance) {

        int thisLevel = getLevel();
@@ -423,6 +409,9 @@ public class PPPCodeLeafCell extends VoronoiLeafCell<PPPCodeObject> implements M 
     */

    @Override

    public boolean split(SplitConfiguration splitConfiguration) throws AlgorithmMethodException {

        if (! isValid()) {

            return false;

        }

        split(getAllObjects(), 0);

        return true;

    }

src/main/java/pppcodes/processors/ApproxNavProcessorRefinement.java

+5 −2
Original line number Diff line number Diff line
@@ -90,13 +90,16 @@ public class ApproxNavProcessorRefinement extends ApproxNavProcessor { 
     * @param operation approximate kNN operation (should be the same for approximate range)

     * @param indexes list of sub-indexes

     */

    public ApproxNavProcessorRefinement(IDObjectRAStorage<? extends LocalAbstractObject> idObjectStorage, ApproxKNNQueryOperation operation, List<PPPCodeSingleAlgorithm> indexes) {

    public ApproxNavProcessorRefinement(IDObjectRAStorage idObjectStorage, ApproxKNNQueryOperation operation, List<PPPCodeSingleAlgorithm> indexes) {

        super(operation, indexes, null, new ArrayBlockingQueue<Integer>(MAX_QUEUE_LENGTH));

        this.blockingCandidateFIFO = (BlockingQueue<Integer>) this.candidateSetIDs;

        this.resolvedObjects = new ArrayBlockingQueue<>(MAX_QUEUE_LENGTH);

        this.currentlyReadObjects = new ArrayBlockingQueue<>(16);

        this.idObjectStorage = idObjectStorage;

        this.maxNumberOfAccessed = operation.getParameter(PARAM_NR_OF_ACCESSED, Integer.class, DEFAULT_NR_OF_ACCESSED);        

        this.idObjectStorage = idObjectStorage;

        

        // create the object filter, if required

        idObjectStorage.preprocessObject(operation.getQueryObject());

    }



    /**