* modifications compliant with new M-Index v. 2.8.0 (renaming of PPCalculator to MIndexInitalizer) (e6dd9e93) · Commits · disa / public / PPP-Codes

pom.xml

+5 −10

Original line number	Diff line number	Diff line
		@@ -4,7 +4,7 @@

		<groupId>mindex</groupId>
		<artifactId>ppp-codes</artifactId>
		<version>1.1</version>
		<version>1.2</version>
		<packaging>jar</packaging>

		<name>ppp-codes</name>
		@@ -62,23 +62,18 @@
		<version>1.3</version>
		<scope>test</scope>
		</dependency>
		<dependency>
		<groupId>${project.groupId}</groupId>
		<artifactId>mindex</artifactId>
		<version>2.7.1-DEVEL</version>
		</dependency>
		<dependency>
		<groupId>net.sf.trove4j</groupId>
		<artifactId>trove4j</artifactId>
		<version>3.0.3</version>
		</dependency>
		<dependency>
		<groupId>messif</groupId>
		<artifactId>messif-private</artifactId>
		<version>1.0</version>
		<groupId>${project.groupId}</groupId>
		<artifactId>mindex</artifactId>
		<version>2.8.0-DEVEL</version>
		</dependency>
		</dependencies>
		<description>PPP-Codes index and search algorithm for approximate metric-based search.
		Version 1.1: simple and hopefully efficient id-object index.
		Version 1.2: clearly separated string-int translation and (optional) refinement storage
		</description>
		</project>

src/main/java/messif/buckets/index/impl/DiskStorageMemoryIntIndex.java

+27 −128

Original line number	Diff line number	Diff line
		/*
		* This file is part of M-Index library: http://disa.fi.muni.cz/results/software/ppp-codes/
		* This file is part of PPP-Codes library: http://disa.fi.muni.cz/results/software/ppp-codes/
		*
		* PPP-Codes library is free software: you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation, either version 3 of the License, or
		* (at your option) any later version.
		*
		* M-Index library is distributed in the hope that it will be useful,
		* PPP-Codes library is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with M-Index library. If not, see <http://www.gnu.org/licenses/>.
		* along with PPP-Codes library. If not, see <http://www.gnu.org/licenses/>.
		*/
		package messif.buckets.index.impl;

		import java.io.File;
		import java.io.IOException;
		import java.io.Serializable;
		import java.lang.reflect.Field;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collection;
		import java.util.Iterator;
		import java.util.List;
		import java.util.logging.Level;
		import java.util.logging.Logger;
		import messif.buckets.BucketStorageException;
		import messif.buckets.index.Search;
		import messif.buckets.storage.impl.DiskStorage;
		import messif.objects.LocalAbstractObject;
		import mindex.MetricIndexes;
		import messif.objects.keys.IntegerKey;

		/**
		* Implementation of disk (long) index that stores the indexed data in a sorted array and keeps the
		@@ -52,9 +48,6 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		/** Storage associated with this index */
		private DiskStorage<LocalAbstractObject> storage;

		/** Copies of the storage for faster reading - the read load is spread equally */
		private transient List<DiskStorage<LocalAbstractObject>> storageCopies = null;

		/** Ordered linked index of addresses into the storage */
		private transient List<IntKeyAddressPair> dynamicIndex;

		@@ -76,32 +69,6 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		this.staticPositions = new long [0];
		}

		/**
		* Creates a new instance of LongStorageMemoryIndex for the specified storage.
		* @param storage the storage to associate with this index
		* @param comparator the comparator imposing natural order of this index
		*/
		public DiskStorageMemoryIntIndex(LongStorageMemoryIndex<Integer, LocalAbstractObject> oldIndex) {
		try {
		Field storageField = LongStorageMemoryIndex.class.getDeclaredField("storage");
		storageField.setAccessible(true);
		this.storage = (DiskStorage<LocalAbstractObject>) storageField.get(oldIndex);

		this.dynamicIndex = new ArrayList<>(MAX_DYNAMIC_LENGTH);
		this.staticIndex = new int [oldIndex.size()];
		this.staticPositions = new long [oldIndex.size()];

		// init this index from the original index data
		for (int i = 0; i < staticIndex.length; i++) {
		LongStorageMemoryIndex.KeyAddressPair<Integer> keyAddress = oldIndex.get(i);
		staticIndex[i] = keyAddress.key;
		staticPositions[i] = keyAddress.position;
		}

		} catch (NoSuchFieldException \| SecurityException \| IllegalArgumentException \| IllegalAccessException ex) {
		Logger.getLogger(DiskStorageMemoryIntIndex.class.getName()).log(Level.SEVERE, null, ex);
		}
		}

		@Override
		public void finalize() throws Throwable {
		@@ -114,17 +81,6 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		storage = null;
		}

		public boolean addStorageCopy(File file) throws IOException {
		if (! file.canRead()) {
		return false;
		}
		if (storageCopies == null) {
		storageCopies = new ArrayList<>();
		}
		storageCopies.add(new DiskStorage<>(storage, file));
		return true;
		}

		// ******************** (De)serialization methods ************************ //
		private void writeObject(java.io.ObjectOutputStream out) throws IOException {
		clearDynamicIndex();
		@@ -202,9 +158,10 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		// **************** Index access methods **************** //

		/**
		* Given an integer key, this methods returns corresponding object from the storage or null (if not found).
		* @param key object integer key
		* @return corresponding object from the storage or null (if not found)
		* Given a set of integer keys, this methods returns corresponding objects from the storage (not necessarily
		* in given order).
		* @param keys object integer keys
		* @return iterator corresponding objects from the storage (not necessarily in the same order)
		*/
		public Iterator<LocalAbstractObject> getObjects(Collection<Integer> keys) {
		long positions [] = new long [keys.size()];
		@@ -218,80 +175,8 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		positions[index ++] = dynamicIndex.get(keyIndex).position;
		}
		}
		if (storageCopies == null \|\| storageCopies.isEmpty() \|\| index <= 1) {
		return storage.read((positions.length == index) ? positions : Arrays.copyOf(positions, index));
		}
		List<Iterator<LocalAbstractObject>> subIterators = new ArrayList<>(storageCopies.size() + 1);
		int indexesStep = index / (storageCopies.size() + 1);
		for (int i = 0; i < storageCopies.size() + 1; i++) {
		subIterators.add(getStorageCopy(i).read(Arrays.copyOfRange(positions, indexesStep * i,
		(i != storageCopies.size()) ? (indexesStep * (i+1)) : Math.max(indexesStep * (i+1), index) )));
		}
		return new ItOverIts(subIterators);
		}

		private DiskStorage<LocalAbstractObject> getStorageCopy(int index) {
		if (index < 0 \|\| index > storageCopies.size()) {
		return null;
		}
		if (index < storageCopies.size()) {
		return storageCopies.get(index);
		}
		return storage;
		}

		/**
		* Iterator over iterators that reads always one object from each iterator and
		* then goes to the following iterator unless all are fully read.
		*/
		protected static class ItOverIts implements Iterator<LocalAbstractObject> {

		private final List<Iterator<LocalAbstractObject>> subIterators;

		int currentId = -1;

		public ItOverIts(List<Iterator<LocalAbstractObject>> subIterators) {
		this.subIterators = subIterators;
		setNextNonemptyIt();
		}

		private void setNextNonemptyIt() {
		if (subIterators.isEmpty()) {
		return;
		}
		if (++ currentId >= subIterators.size()) {
		currentId = 0;
		}
		while (! subIterators.get(currentId).hasNext()) {
		subIterators.remove(currentId);
		if (subIterators.isEmpty()) {
		return;
		}
		if (currentId >= subIterators.size()) {
		currentId = 0;
		}
		}
		}

		@Override
		public boolean hasNext() {
		return ! subIterators.isEmpty();
		}

		@Override
		public LocalAbstractObject next() {
		try {
		return (subIterators.get(currentId).next());
		} finally {
		setNextNonemptyIt();
		}
		}

		@Override
		public void remove() {
		throw new UnsupportedOperationException("This is read-only iterator."); //To change body of generated methods, choose Tools \| Templates.
		}
		}

		/**
		* Returns the maximal currently indexed key.
		@@ -314,7 +199,7 @@ public class DiskStorageMemoryIntIndex implements Serializable {


		protected int extractKey(LocalAbstractObject object) {
		return MetricIndexes.getIntegerID(object);
		return DiskStorageMemoryIntIndex.getIntegerID(object);
		}

		/**
		@@ -342,8 +227,6 @@ public class DiskStorageMemoryIntIndex implements Serializable {

		/**
		* Class encapsulating the key and long position in the storage.
		*
		* @param <K> the type of keys this index is ordered by
		*/
		protected static class IntKeyAddressPair {

		@@ -357,4 +240,20 @@ public class DiskStorageMemoryIntIndex implements Serializable {
		}
		}

		/**
		* Reads and returns the integer ID supposedly stored in the {@link IntegerKey} within the given object. If not
		* stored, the string locator is tried to be converted to integer. If not, the locator hash code is returned.
		* @param object object to get integer ID for
		* @return integer ID for given object
		*/
		public static int getIntegerID(LocalAbstractObject object) {
		try {
		if (object.getObjectKey() instanceof IntegerKey) {
		return ((IntegerKey) object.getObjectKey()).key;
		}
		return Integer.valueOf(object.getLocatorURI());
		} catch (NumberFormatException ex) {
		return object.getLocatorURI().hashCode();
		}
		}
		}

src/main/java/messif/objects/impl/EMDAux.java

deleted100644 → 0

+0 −402

Original line number	Diff line number	Diff line
		/*
		* This file is part of M-Index library: http://disa.fi.muni.cz/results/software/ppp-codes/
		*
		* PPP-Codes library is free software: you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation, either version 3 of the License, or
		* (at your option) any later version.
		*
		* M-Index library is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with M-Index library. If not, see <http://www.gnu.org/licenses/>.
		*/
		package messif.objects.impl;

		import java.util.ArrayList;
		import java.util.LinkedList;
		import java.util.List;


		/**
		* @author David Novak, Masaryk University, Brno, Czech Republic, novak.david@gmail.com
		*
		*/
		class Edge {
		Edge(int to, long cost) {
		_to = to;
		_cost = cost;
		}

		int _to;
		long _cost;
		}

		class Edge0 {
		Edge0(int to, long cost, long flow) {
		_to = to;
		_cost = cost;
		_flow = flow;
		}

		int _to;
		long _cost;
		long _flow;
		}

		class Edge1 {
		Edge1(int to, long reduced_cost) {
		_to = to;
		_reduced_cost = reduced_cost;
		}

		int _to;
		long _reduced_cost;
		}

		class Edge2 {
		Edge2(int to, long reduced_cost, long residual_capacity) {
		_to = to;
		_reduced_cost = reduced_cost;
		_residual_capacity = residual_capacity;
		}

		int _to;
		long _reduced_cost;
		long _residual_capacity;
		}

		class Edge3 {
		Edge3() {
		_to = 0;
		_dist = 0;
		}

		Edge3(int to, long dist) {
		_to = to;
		_dist = dist;
		}

		int _to;
		long _dist;
		}

		class MinCostFlow {

		int numNodes;
		//List<Integer> nodesToQ;
		int nodesToQ [];

		// e - supply(positive) and demand(negative).
		// c[i] - edges that goes from node i. first is the second nod
		// x - the flow is returned in it
		long compute(long [] e, List<List<Edge>> c, List<List<Edge0>> x) {
		assert (e.length == c.size());
		assert (x.size() == c.size());

		numNodes = e.length;
		nodesToQ = new int[numNodes];

		// init flow
		for (int from = 0; from < numNodes; ++from) {
		for (Edge it : c.get(from)) {
		x.get(from).add(new Edge0(it._to, it._cost, 0));
		x.get(it._to).add(new Edge0(from, -it._cost, 0));
		}
		}

		// reduced costs for forward edges (c[i,j]-pi[i]+pi[j])
		// Note that for forward edges the residual capacity is infinity
		List<List<Edge1>> rCostForward = new ArrayList<>();
		for (int i = 0; i < numNodes; i++) {
		rCostForward.add(new LinkedList<Edge1>());
		}
		for (int from = 0; from < numNodes; ++from) {
		for (Edge it : c.get(from)) {
		rCostForward.get(from).add(new Edge1(it._to, it._cost));
		}
		}

		// reduced costs and capacity for backward edges
		// (c[j,i]-pi[j]+pi[i])
		// Since the flow at the beginning is 0, the residual capacity is
		// also zero
		List<List<Edge2>> rCostCapBackward = new ArrayList<>();
		for (int i = 0; i < numNodes; i++) {
		rCostCapBackward.add(new LinkedList<Edge2>());
		}
		for (int from = 0; from < numNodes; ++from) {
		for (Edge it : c.get(from)) {
		rCostCapBackward.get(it._to).add(
		new Edge2(from, -it._cost, 0));
		}
		}

		// Max supply TODO:demand?, given U?, optimization-> min out of
		// demand,supply
		long U = 0;
		for (int i = 0; i < numNodes; i++) {
		if (e[i] > U)
		U = e[i];
		}
		// long delta = (long) (Math.pow(2.0,
		// Math.ceil(Math.log((float) (U)) / Math.log(2.0))));

		// List<Long> d = new List<Long>();
		// List<Integer> prev = new List<Integer>();
		// for (int i = 0; i < numNodes; i++) {
		// d.add(0l);
		// prev.add(0);
		// }
		long [] d = new long [numNodes];
		int [] prev = new int [numNodes];
		long delta;
		while (true) { // until we break when S or T is empty
		long maxSupply = 0;
		int k = 0;
		for (int i = 0; i < numNodes; i++) {
		if (e[i] > 0) {
		if (maxSupply < e[i]) {
		maxSupply = e[i];
		k = i;
		}
		}
		}
		if (maxSupply == 0)
		break;
		delta = maxSupply;

		int[] l = new int[1];
		computeShortestPath(d, prev, k, rCostForward, rCostCapBackward,
		e, l);

		// find delta (minimum on the path from k to l)
		// delta= e[k];
		// if (-e[l]<delta) delta= e[k];
		int to = l[0];
		do {
		int from = prev[to];
		assert (from != to);

		// residual
		int itccb = 0;
		while ((itccb < rCostCapBackward.get(from).size())
		&& (rCostCapBackward.get(from).get(itccb)._to != to)) {
		itccb++;
		}
		if (itccb < rCostCapBackward.get(from).size()) {
		if (rCostCapBackward.get(from).get(itccb)._residual_capacity < delta)
		delta = rCostCapBackward.get(from).get(itccb)._residual_capacity;
		}

		to = from;
		} while (to != k);

		// augment delta flow from k to l (backwards actually...)
		to = l[0];
		do {
		int from = prev[to];
		assert (from != to);

		// TODO - might do here O(n) can be done in O(1)
		int itx = 0;
		while (x.get(from).get(itx)._to != to) {
		itx++;
		}
		x.get(from).get(itx)._flow += delta;

		// update residual for backward edges
		int itccb = 0;
		while ((itccb < rCostCapBackward.get(to).size())
		&& (rCostCapBackward.get(to).get(itccb)._to != from)) {
		itccb++;
		}
		if (itccb < rCostCapBackward.get(to).size()) {
		rCostCapBackward.get(to).get(itccb)._residual_capacity += delta;
		}
		itccb = 0;
		while ((itccb < rCostCapBackward.get(from).size())
		&& (rCostCapBackward.get(from).get(itccb)._to != to)) {
		itccb++;
		}
		if (itccb < rCostCapBackward.get(from).size()) {
		rCostCapBackward.get(from).get(itccb)._residual_capacity -= delta;
		}

		// update e
		e[to] = e[to] + delta;
		e[from] = e[from] - delta;

		to = from;
		} while (to != k);
		}

		// compute distance from x
		long dist = 0;
		for (int from = 0; from < numNodes; from++) {
		for (Edge0 it : x.get(from)) {
		dist += (it._cost * it._flow);
		}
		}
		return dist;
		}

		void computeShortestPath(long [] d, int [] prev,
		int from, List<List<Edge1>> costForward,
		List<List<Edge2>> costBackward, long [] e, int[] l) {
		// Making heap (all inf except 0, so we are saving comparisons...)
		List<Edge3> Q = new ArrayList<>(numNodes);
		for (int i = 0; i < numNodes; i++) {
		Q.add(new Edge3());
		}

		Q.get(0)._to = from;
		nodesToQ[from] = 0;
		Q.get(0)._dist = 0;

		int j = 1;
		// TODO: both of these into a function?
		for (int i = 0; i < from; ++i) {
		Q.get(j)._to = i;
		nodesToQ[i] = j;
		Q.get(j)._dist = Long.MAX_VALUE;
		j++;
		}

		for (int i = from + 1; i < numNodes; i++) {
		Q.get(j)._to = i;
		nodesToQ[i] = j;
		Q.get(j)._dist = Long.MAX_VALUE;
		j++;
		}

		boolean [] finalNodesFlg = new boolean [numNodes];
		// for (int i = 0; i < numNodes; i++) {
		// finalNodesFlg.add(false);
		// }
		do {
		int u = Q.get(0)._to;

		d[u] = Q.get(0)._dist; // final distance
		finalNodesFlg[u] = true;
		if (e[u] < 0) {
		l[0] = u;
		break;
		}

		heapRemoveFirst(Q, nodesToQ);

		// neighbors of u
		for (Edge1 it : costForward.get(u)) {
		assert (it._reduced_cost >= 0);
		long alt = d[u] + it._reduced_cost;
		int v = it._to;
		if ((nodesToQ[v] < Q.size())
		&& (alt < Q.get(nodesToQ[v])._dist)) {
		heapDecreaseKey(Q, nodesToQ, v, alt);
		prev[v] = u;
		}
		}
		for (Edge2 it : costBackward.get(u)) {
		if (it._residual_capacity > 0) {
		assert (it._reduced_cost >= 0);
		long alt = d[u] + it._reduced_cost;
		int v = it._to;
		if ((nodesToQ[v] < Q.size())
		&& (alt < Q.get(nodesToQ[v])._dist)) {
		heapDecreaseKey(Q, nodesToQ, v, alt);
		prev[v] = u;
		}
		}
		}

		} while (Q.size() > 0);

		for (int _from = 0; _from < numNodes; ++_from) {
		for (Edge1 it : costForward.get(_from)) {
		if (finalNodesFlg[_from]) {
		it._reduced_cost += d[_from] - d[l[0]];
		}
		if (finalNodesFlg[it._to]) {
		it._reduced_cost -= d[it._to] - d[l[0]];
		}
		}
		}

		// reduced costs and capacity for backward edges
		// (c[j,i]-pi[j]+pi[i])
		for (int _from = 0; _from < numNodes; ++_from) {
		for (Edge2 it : costBackward.get(_from)) {
		if (finalNodesFlg[_from]) {
		it._reduced_cost += d[_from] - d[l[0]];
		}
		if (finalNodesFlg[it._to]) {
		it._reduced_cost -= d[it._to] - d[l[0]];
		}
		}
		}
		}

		void heapDecreaseKey(List<Edge3> Q, int [] nodes_to_Q,
		int v, long alt) {
		int i = nodes_to_Q[v];
		Q.get(i)._dist = alt;
		while (i > 0 && Q.get(PARENT(i))._dist > Q.get(i)._dist) {
		swapHeap(Q, nodes_to_Q, i, PARENT(i));
		i = PARENT(i);
		}
		}

		void heapRemoveFirst(List<Edge3> Q, int [] nodes_to_Q) {
		swapHeap(Q, nodes_to_Q, 0, Q.size() - 1);
		Q.remove(Q.size() - 1);
		heapify(Q, nodes_to_Q, 0);
		}

		void heapify(List<Edge3> Q, int [] nodes_to_Q, int i) {
		do {
		// TODO: change to loop
		int l = LEFT(i);
		int r = RIGHT(i);
		int smallest;
		if ((l < Q.size()) && (Q.get(l)._dist < Q.get(i)._dist)) {
		smallest = l;
		} else {
		smallest = i;
		}
		if ((r < Q.size()) && (Q.get(r)._dist < Q.get(smallest)._dist)) {
		smallest = r;
		}

		if (smallest == i)
		return;

		swapHeap(Q, nodes_to_Q, i, smallest);
		i = smallest;

		} while (true);
		}

		void swapHeap(List<Edge3> Q, int [] nodesToQ, int i, int j) {
		Edge3 tmp = Q.get(i);
		Q.set(i, Q.get(j));
		Q.set(j, tmp);
		nodesToQ[Q.get(j)._to] = j;
		nodesToQ[Q.get(i)._to] = i;
		}

		int LEFT(int i) {
		return 2 * (i + 1) - 1;
		}

		int RIGHT(int i) {
		return 2 * (i + 1); // 2 * (i + 1) + 1 - 1
		}

		int PARENT(int i) {
		return (i - 1) / 2;
		}
		}
		No newline at end of file

src/main/java/messif/objects/impl/EarthMoverDistance.java

deleted100644 → 0

+0 −338

File deleted.

Preview size limit exceeded, changes collapsed.

src/main/java/messif/objects/impl/FeatureFloatL2Temporary.java

deleted100644 → 0

+0 −71

Original line number	Diff line number	Diff line
		/*
		* This file is part of M-Index library: http://disa.fi.muni.cz/results/software/ppp-codes/
		*
		* PPP-Codes library is free software: you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation, either version 3 of the License, or
		* (at your option) any later version.
		*
		* M-Index library is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with M-Index library. If not, see <http://www.gnu.org/licenses/>.
		*/
		package messif.objects.impl;

		import java.io.IOException;
		import java.io.OutputStream;
		import messif.objects.nio.BinaryInput;
		import messif.objects.nio.BinaryOutput;
		import messif.objects.nio.BinarySerializable;
		import messif.objects.nio.BinarySerializator;

		/**
		*
		* @author David Novak, Masaryk University, Brno, Czech Republic, novak.david@gmail.com
		*/
		public class FeatureFloatL2Temporary implements ObjectFeatureTemporary, BinarySerializable {

		final private float [] values;

		public FeatureFloatL2Temporary(float[] values) {
		this.values = values;
		}

		@Override
		public void writeData(OutputStream stream) throws IOException {
		ObjectFloatVector.writeFloatVector(values, stream, ',', ' ');
		}

		@Override
		public float featureDistance(ObjectFeatureTemporary f) {
		FeatureFloatL2Temporary ffl2 = (FeatureFloatL2Temporary) f;
		float powSum = 0;
		for (int i = 0; i < values.length; i++) {
		float dif = (values[i] - ffl2.values[i]);
		powSum += dif * dif;
		}

		return (float)Math.sqrt(powSum);
		}


		public FeatureFloatL2Temporary(BinaryInput input, BinarySerializator serializator) throws IOException {
		this.values = serializator.readFloatArray(input);
		}


		@Override
		public int getBinarySize(BinarySerializator serializator) {
		return serializator.getBinarySize(values);
		}

		@Override
		public int binarySerialize(BinaryOutput output, BinarySerializator serializator) throws IOException {
		return serializator.write(output, values);
		}

		}