diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 8cf2a8aacbece..20a85c0593f31 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -2,7 +2,9 @@ steps: - trigger: apache-lucene-build-snapshot label: Trigger pipeline to build lucene snapshot key: lucene-build - if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "" + if: (build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "") + build: + branch: branch_10x - wait - label: Upload and update lucene snapshot command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 29cd82ca60d81..2e36aadbf4c5e 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.2.1-snapshot-ae6484f43e6 +lucene = 10.3.0-snapshot-96553b7aff7 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 5191b60f1f8c9..93ee3850be04f 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -61,13 +61,6 @@ static List systemJvmOptions(Settings nodeSettings, final Map - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index ef1a7d895c234..8ad85d0168349 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -40,7 +40,6 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapReflectionUtils; import org.elasticsearch.jdk.JarHell; import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.monitor.jvm.HotThreads; @@ -234,9 +233,7 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { // RequestHandlerRegistry and MethodHandlers classes do nontrivial static initialization which should always succeed but load // it now (before SM) to be sure RequestHandlerRegistry.class, - MethodHandlers.class, - // Ensure member access and reflection lookup are as expected - OffHeapReflectionUtils.class + MethodHandlers.class ); // load the plugin Java modules and layers now for use in entitlements @@ -398,7 +395,7 @@ private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { private static void ensureInitialized(Class... classes) { for (final var clazz : classes) { try { - MethodHandles.lookup().ensureInitialized(clazz); + MethodHandles.publicLookup().ensureInitialized(clazz); } catch (IllegalAccessException unexpected) { throw new AssertionError(unexpected); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index b1cf51d89ad31..5843c562dd5ff 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -163,6 +163,8 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1); public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1); public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1); + + public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index 29f62b64764a9..325188624a2f4 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -29,8 +29,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.Map; @@ -105,7 +103,7 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } - static class ES813FlatVectorReader extends KnnVectorsReader implements OffHeapStats { + static class ES813FlatVectorReader extends KnnVectorsReader { private final FlatVectorsReader reader; @@ -152,13 +150,13 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits } @Override - public void close() throws IOException { - reader.close(); + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); } @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return OffHeapByteSizeUtils.getOffHeapByteSize(reader, fieldInfo); + public void close() throws IOException { + reader.close(); } } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index 8dd4f686a6dea..4636a37d14f53 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -27,8 +27,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.Map; @@ -113,7 +111,7 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } - public static class ES813FlatVectorReader extends KnnVectorsReader implements OffHeapStats { + public static class ES813FlatVectorReader extends KnnVectorsReader { private final FlatVectorsReader reader; @@ -160,13 +158,13 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits } @Override - public void close() throws IOException { - reader.close(); + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); } @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return OffHeapByteSizeUtils.getOffHeapByteSize(reader, fieldInfo); + public void close() throws IOException { + reader.close(); } } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 84981c3d2a1be..204d9c6e9e630 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -34,8 +34,6 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues; import org.apache.lucene.util.quantization.QuantizedVectorsReader; import org.apache.lucene.util.quantization.ScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import org.elasticsearch.simdvec.VectorScorerFactory; import org.elasticsearch.simdvec.VectorSimilarityType; @@ -177,7 +175,7 @@ public long ramBytesUsed() { } } - static final class ES814ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader, OffHeapStats { + static final class ES814ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader { final Lucene99ScalarQuantizedVectorsReader delegate; @@ -233,7 +231,7 @@ public long ramBytesUsed() { @Override public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return OffHeapByteSizeUtils.getOffHeapByteSize(delegate, fieldInfo); + return delegate.getOffHeapByteSize(fieldInfo); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java index f809fd81bbd52..5f290a72ba833 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.codec.vectors.es816; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; import org.apache.lucene.index.ByteVectorValues; @@ -44,13 +45,10 @@ import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.elasticsearch.index.codec.vectors.BQVectorUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -60,7 +58,7 @@ * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @SuppressForbidden(reason = "Lucene classes") -public class ES816BinaryQuantizedVectorsReader extends FlatVectorsReader implements OffHeapStats { +public class ES816BinaryQuantizedVectorsReader extends FlatVectorsReader { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES816BinaryQuantizedVectorsReader.class); @@ -259,15 +257,14 @@ public long ramBytesUsed() { @Override public Map getOffHeapByteSize(FieldInfo fieldInfo) { - Objects.requireNonNull(fieldInfo); - var raw = OffHeapByteSizeUtils.getOffHeapByteSize(rawVectorsReader, fieldInfo); - var fieldEntry = fields.get(fieldInfo.name); - if (fieldEntry == null) { + var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); + FieldEntry fe = fields.get(fieldInfo.name); + if (fe == null) { assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; return raw; } - var quant = Map.of(VECTOR_DATA_EXTENSION, fieldEntry.vectorDataLength()); - return OffHeapByteSizeUtils.mergeOffHeapByteSizeMaps(raw, quant); + var quant = Map.of(VECTOR_DATA_EXTENSION, fe.vectorDataLength()); + return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); } public float[] getCentroid(String field) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java index 61426d9f33008..0d087cb5d73c9 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Map; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -171,6 +172,12 @@ public long ramBytesUsed() { return SHALLOW_SIZE + fields.ramBytesUsed(); } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + final FieldEntry entry = getFieldEntryOrThrow(fieldInfo.name); + return Map.of(DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength()); + } + @Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(vectorData); @@ -187,12 +194,17 @@ public FlatVectorsReader getMergeInstance() { } } - private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { + private FieldEntry getFieldEntryOrThrow(String field) { final FieldInfo info = fieldInfos.fieldInfo(field); - final FieldEntry fieldEntry; - if (info == null || (fieldEntry = fields.get(info.number)) == null) { + final FieldEntry entry; + if (info == null || (entry = fields.get(info.number)) == null) { throw new IllegalArgumentException("field=\"" + field + "\" not found"); } + return entry; + } + + private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { + final FieldEntry fieldEntry = getFieldEntryOrThrow(field); if (fieldEntry.vectorEncoding != expectedEncoding) { throw new IllegalArgumentException( "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + expectedEncoding diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java index ac707d155ea3f..8e547f9b3fa20 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.codec.vectors.es818; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; import org.apache.lucene.index.ByteVectorValues; @@ -45,13 +46,10 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.elasticsearch.index.codec.vectors.BQVectorUtils; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -61,7 +59,7 @@ * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @SuppressForbidden(reason = "Lucene classes") -public class ES818BinaryQuantizedVectorsReader extends FlatVectorsReader implements OffHeapStats { +public class ES818BinaryQuantizedVectorsReader extends FlatVectorsReader { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES818BinaryQuantizedVectorsReader.class); @@ -259,15 +257,14 @@ public long ramBytesUsed() { @Override public Map getOffHeapByteSize(FieldInfo fieldInfo) { - Objects.requireNonNull(fieldInfo); - var raw = OffHeapByteSizeUtils.getOffHeapByteSize(rawVectorsReader, fieldInfo); - var fieldEntry = fields.get(fieldInfo.name); - if (fieldEntry == null) { + var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); + FieldEntry fe = fields.get(fieldInfo.name); + if (fe == null) { assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; return raw; } - var quant = Map.of(VECTOR_DATA_EXTENSION, fieldEntry.vectorDataLength()); - return OffHeapByteSizeUtils.mergeOffHeapByteSizeMaps(raw, quant); + var quant = Map.of(VECTOR_DATA_EXTENSION, fe.vectorDataLength()); + return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); } public float[] getCentroid(String field) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java deleted file mode 100644 index bf47564c11b3a..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.codecs.KnnVectorsReader; -import org.elasticsearch.core.SuppressForbidden; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.security.AccessController; -import java.security.PrivilegedAction; - -/** - * Reflective access to unwrap non-accessible delegate in AssertingKnnVectorsReader. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class AssertingKnnVectorsReaderReflect { - - @SuppressForbidden(reason = "static type is not accessible") - public static KnnVectorsReader unwrapAssertingReader(KnnVectorsReader reader) { - try { - if (ASSERTING_ASSERT_KNN_READER_CLS != null && ASSERTING_ASSERT_KNN_READER_CLS.isAssignableFrom(reader.getClass())) { - return (KnnVectorsReader) GET_VECTOR_INDEX_LENGTH_HANDLE.invoke(reader); - } - } catch (Throwable t) { - handleThrowable(t); - } - return reader; - } - - private static final Class ASSERTING_ASSERT_KNN_READER_CLS = getAssertingReaderOrNull(); - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE = getDelegateFieldHandle(); - - private static Class getAssertingReaderOrNull() { - try { - return Class.forName("org.apache.lucene.tests.codecs.asserting.AssertingKnnVectorsFormat$AssertingKnnVectorsReader"); - } catch (ClassNotFoundException e) { - return null; - } - } - - private static MethodHandle getDelegateFieldHandle() { - try { - var cls = getAssertingReaderOrNull(); - if (cls == null) { - return MethodHandles.throwException(KnnVectorsReader.class, AssertionError.class); - } - var lookup = privilegedPrivateLookupIn(cls, MethodHandles.lookup()); - return lookup.findGetter(cls, "delegate", KnnVectorsReader.class); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } - - @SuppressWarnings("removal") - static MethodHandles.Lookup privilegedPrivateLookupIn(Class cls, MethodHandles.Lookup lookup) throws IllegalAccessException { - PrivilegedAction pa = () -> { - try { - return MethodHandles.privateLookupIn(cls, lookup); - } catch (IllegalAccessException e) { - throw new AssertionError("should not happen, check opens", e); - } - }; - return AccessController.doPrivileged(pa); - } - - static void handleThrowable(Throwable t) { - if (t instanceof Error error) { - throw error; - } else if (t instanceof RuntimeException runtimeException) { - throw runtimeException; - } else { - throw new AssertionError(t); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java deleted file mode 100644 index dcd9fc3b12737..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader; - -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Static utility methods to help retrieve desired off-heap vector index size. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class OffHeapByteSizeUtils { - - private OffHeapByteSizeUtils() {} // no instances - - public static Map getOffHeapByteSize(KnnVectorsReader reader, FieldInfo fieldInfo) { - reader = AssertingKnnVectorsReaderReflect.unwrapAssertingReader(reader); - switch (reader) { - case OffHeapStats offHeapStats -> { - return offHeapStats.getOffHeapByteSize(fieldInfo); - } - case Lucene99HnswVectorsReader hnswVectorsReader -> { - var graph = OffHeapReflectionUtils.getOffHeapByteSizeL99HNSW(hnswVectorsReader, fieldInfo); - var flat = getOffHeapByteSize(OffHeapReflectionUtils.getFlatVectorsReaderL99HNSW(hnswVectorsReader), fieldInfo); - return mergeOffHeapByteSizeMaps(graph, flat); - } - case Lucene99ScalarQuantizedVectorsReader scalarQuantizedVectorsReader -> { - var quant = OffHeapReflectionUtils.getOffHeapByteSizeSQ(scalarQuantizedVectorsReader, fieldInfo); - var raw = getOffHeapByteSize(OffHeapReflectionUtils.getFlatVectorsReaderSQ(scalarQuantizedVectorsReader), fieldInfo); - return mergeOffHeapByteSizeMaps(quant, raw); - } - case Lucene99FlatVectorsReader flatVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeF99FLT(flatVectorsReader, fieldInfo); - } - case DirectIOLucene99FlatVectorsReader flatVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeF99FLT(flatVectorsReader, fieldInfo); - } - case Lucene95HnswVectorsReader lucene95HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL95HNSW(lucene95HnswVectorsReader, fieldInfo); - } - case Lucene94HnswVectorsReader lucene94HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL94HNSW(lucene94HnswVectorsReader, fieldInfo); - } - case Lucene92HnswVectorsReader lucene92HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL92HNSW(lucene92HnswVectorsReader, fieldInfo); - } - case Lucene91HnswVectorsReader lucene91HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL91HNSW(lucene91HnswVectorsReader, fieldInfo); - } - case Lucene90HnswVectorsReader lucene90HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL90HNSW(lucene90HnswVectorsReader, fieldInfo); - } - case null, default -> { - assert false : "unexpected reader:" + reader; - } - } - return Map.of(); - } - - /** - * Merges the Maps returned by getOffHeapByteSize(FieldInfo). - * - *

This method is a convenience for aggregating the desired off-heap memory requirements for - * several fields. The keys in the returned map are a union of the keys in the given maps. Entries - * with the same key are summed. - */ - public static Map mergeOffHeapByteSizeMaps(Map map1, Map map2) { - return Stream.of(map1, map2) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java deleted file mode 100644 index 599a205508385..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.VectorEncoding; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Map; - -import static java.lang.invoke.MethodType.methodType; - -/** - * Reflective access to non-accessible members of Lucene's KnnVectorsReader implementations. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class OffHeapReflectionUtils { - - private OffHeapReflectionUtils() {} - - static final String FLAT_VECTOR_DATA_EXTENSION = "vec"; - static final String SQ_VECTOR_INDEX_EXTENSION = "veq"; - static final String HNSW_VECTOR_INDEX_EXTENSION = "vex"; - - private static final MethodHandle GET_FIELD_ENTRY_HNDL_SQ; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_SQ; - private static final VarHandle RAW_VECTORS_READER_HNDL_SQ; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L99FLT; - private static final MethodHandle VECTOR_DATA_LENGTH_HANDLE_L99FLT; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_DIOL99FLT; - private static final MethodHandle VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L99HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW; - private static final VarHandle FLAT_VECTORS_READER_HNDL_L99HNSW; - - static final Class L99_SQ_VR_CLS = Lucene99ScalarQuantizedVectorsReader.class; - static final Class L99_FLT_VR_CLS = Lucene99FlatVectorsReader.class; - static final Class DIOL99_FLT_VR_CLS = DirectIOLucene99FlatVectorsReader.class; - static final Class L99_HNSW_VR_CLS = Lucene99HnswVectorsReader.class; - - // old codecs - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L90HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L91HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L92HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L94HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L95HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW; - - static final Class L90_HNSW_VR_CLS = Lucene90HnswVectorsReader.class; - static final Class L91_HNSW_VR_CLS = Lucene91HnswVectorsReader.class; - static final Class L92_HNSW_VR_CLS = Lucene92HnswVectorsReader.class; - static final Class L94_HNSW_VR_CLS = Lucene94HnswVectorsReader.class; - static final Class L95_HNSW_VR_CLS = Lucene95HnswVectorsReader.class; - - static { - try { - // Lucene99ScalarQuantizedVectorsReader - var cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader$FieldEntry"); - var lookup = privilegedPrivateLookupIn(L99_SQ_VR_CLS, MethodHandles.lookup()); - var mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HNDL_SQ = lookup.findVirtual(L99_SQ_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_DATA_LENGTH_HANDLE_SQ = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - RAW_VECTORS_READER_HNDL_SQ = lookup.findVarHandle(L99_SQ_VR_CLS, "rawVectorsReader", FlatVectorsReader.class); - // Lucene99FlatVectorsReader - cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L99_FLT_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L99FLT = lookup.findVirtual(L99_FLT_VR_CLS, "getFieldEntry", mt); - VECTOR_DATA_LENGTH_HANDLE_L99FLT = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // DirectIOLucene99FlatVectorsReader - cls = Class.forName("org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(DIOL99_FLT_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_DIOL99FLT = lookup.findVirtual(DIOL99_FLT_VR_CLS, "getFieldEntry", mt); - VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene99HnswVectorsReader - cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L99_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L99HNSW = lookup.findVirtual(L99_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - lookup = privilegedPrivateLookupIn(L99_HNSW_VR_CLS, MethodHandles.lookup()); - FLAT_VECTORS_READER_HNDL_L99HNSW = lookup.findVarHandle(L99_HNSW_VR_CLS, "flatVectorsReader", FlatVectorsReader.class); - // Lucene90HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L90_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L90HNSW = lookup.findVirtual(L90_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW = lookup.findVirtual(cls, "indexDataLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene91HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L91_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L91HNSW = lookup.findVirtual(L91_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene92HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L92_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L92HNSW = lookup.findVirtual(L92_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene94HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L94_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L94HNSW = lookup.findVirtual(L94_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene95HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L95_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L95HNSW = lookup.findVirtual(L95_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeSQ(Lucene99ScalarQuantizedVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HNDL_SQ.invoke(reader, fieldInfo.name); - long len = (long) GET_VECTOR_DATA_LENGTH_HANDLE_SQ.invoke(entry); - return Map.of(SQ_VECTOR_INDEX_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - static FlatVectorsReader getFlatVectorsReaderSQ(Lucene99ScalarQuantizedVectorsReader reader) { - return (FlatVectorsReader) RAW_VECTORS_READER_HNDL_SQ.get(reader); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeF99FLT(Lucene99FlatVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L99FLT.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) VECTOR_DATA_LENGTH_HANDLE_L99FLT.invoke(entry); - return Map.of(FLAT_VECTOR_DATA_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeF99FLT(DirectIOLucene99FlatVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_DIOL99FLT.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT.invoke(entry); - return Map.of(FLAT_VECTOR_DATA_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL99HNSW(Lucene99HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L99HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - static FlatVectorsReader getFlatVectorsReaderL99HNSW(Lucene99HnswVectorsReader reader) { - return (FlatVectorsReader) FLAT_VECTORS_READER_HNDL_L99HNSW.get(reader); - } - - // old codecs - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL90HNSW(Lucene90HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L90HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL91HNSW(Lucene91HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L91HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL92HNSW(Lucene92HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L92HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL94HNSW(Lucene94HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L94HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL95HNSW(Lucene95HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L95HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressWarnings("removal") - private static MethodHandles.Lookup privilegedPrivateLookupIn(Class cls, MethodHandles.Lookup lookup) { - PrivilegedAction pa = () -> { - try { - return MethodHandles.privateLookupIn(cls, lookup); - } catch (IllegalAccessException e) { - throw new AssertionError("should not happen, check opens", e); - } - }; - return AccessController.doPrivileged(pa); - } - - private static void handleThrowable(Throwable t) { - if (t instanceof Error error) { - throw error; - } else if (t instanceof RuntimeException runtimeException) { - throw runtimeException; - } else { - throw new AssertionError(t); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java deleted file mode 100644 index 79eb118f389cc..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.index.FieldInfo; - -import java.util.Map; - -/** - * Common interface to unify offHeapByteSize in ES' KnnVectorsReader implementations. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public interface OffHeapStats { - - Map getOffHeapByteSize(FieldInfo fieldInfo); -} diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index afd0ceb6d066d..9c34f4855d465 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -62,7 +62,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.FieldInfosWithUsages; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.elasticsearch.index.mapper.DocumentParser; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; @@ -334,7 +333,7 @@ private DenseVectorStats getDenseVectorStats(final LeafReader atomicReader, List if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { vectorsReader = fieldsReader.getFieldReader(info.name); } - Map offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(vectorsReader, info); + Map offHeap = vectorsReader.getOffHeapByteSize(info); offHeapStats.put(info.name, offHeap); } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java b/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java index 6e74647899080..e60f325e2f5b1 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java @@ -9,11 +9,11 @@ package org.elasticsearch.index.shard; +import org.apache.lucene.codecs.KnnVectorsReader; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -92,7 +92,7 @@ public void add(DenseVectorStats other) { } else { this.offHeapStats = Stream.of(this.offHeapStats, other.offHeapStats) .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, OffHeapByteSizeUtils::mergeOffHeapByteSizeMaps)); + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, KnnVectorsReader::mergeOffHeapByteSizeMaps)); } } } diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 4f3bc1f92060b..55abdc84fc8fb 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -32,10 +32,6 @@ grant codeBase "${codebase.elasticsearch}" { // for plugin api dynamic settings instances permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect"; - - // For vector off-heap statistics, remove in Lucene 10.3 - permission java.lang.RuntimePermission "accessDeclaredMembers"; - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; //// Very special jar permissions: diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java index 80b2d5232ca65..9ea13750d8cd7 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; @@ -66,7 +65,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1, offHeap.size()); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java index 99c60c7bcc7f3..6a29bf08ae9a5 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; @@ -66,7 +65,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veq") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java index be715c8ab9175..b064321a70ae8 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java @@ -30,7 +30,6 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.nio.file.Path; @@ -199,7 +198,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java index b04e74adc53ae..c770a62479d38 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.junit.Before; import java.io.IOException; @@ -60,7 +59,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(1, offHeap.size()); assertTrue(offHeap.get("vec") > 0L); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java index 2e30fa0d16fa2..544dccc73c070 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.junit.Before; import java.io.IOException; @@ -60,7 +59,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertTrue(offHeap.get("vec") > 0L); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java index a18d86128e90f..6aec7b595ce8b 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java @@ -45,7 +45,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.index.codec.vectors.BQVectorUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Locale; @@ -200,7 +199,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veb") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java index 5658935469302..a80240e228efc 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java @@ -41,7 +41,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.util.SameThreadExecutorService; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Arrays; @@ -147,7 +146,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java index 4e431cbe71eab..26bd905270f83 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.index.codec.vectors.BQVectorUtils; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Locale; @@ -200,7 +199,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veb") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java index 63af621fafd31..cf7b710ef3093 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java @@ -41,7 +41,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.util.SameThreadExecutorService; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Arrays; @@ -151,7 +150,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 1f74668158e0e..0d7f16211aa51 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -55,6 +55,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; @@ -567,12 +568,13 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception { // size is lower than terminate_after context.setSize(5); // track_total_hits is lower than terminate_after - context.trackTotalHitsUpTo(randomIntBetween(1, 6)); + int trackTotalHits = randomIntBetween(1, 6); + context.trackTotalHitsUpTo(trackTotalHits); QueryPhase.executeQuery(context); // depending on docs distribution we may or may not be able to honor terminate_after: low scoring hits are skipped via // setMinCompetitiveScore, which bypasses terminate_after until the next leaf collector is pulled, when that happens. assertThat(context.queryResult().terminatedEarly(), either(is(true)).or(is(false))); - assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(7L)); + assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), greaterThanOrEqualTo((long) trackTotalHits)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(5)); } @@ -990,7 +992,10 @@ public void testMinScore() throws Exception { context.trackTotalHitsUpTo(5); QueryPhase.addCollectorsAndSearch(context); - assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value()); + TotalHits totalHits = context.queryResult().topDocs().topDocs.totalHits; + assertThat(totalHits.value(), greaterThanOrEqualTo(5L)); + var expectedRelation = totalHits.value() == 10 ? Relation.EQUAL_TO : Relation.GREATER_THAN_OR_EQUAL_TO; + assertThat(totalHits.relation(), is(expectedRelation)); } }