From b527f593cd845477bfd9f1f4ec8040d7c101c02a Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Sun, 20 Apr 2025 10:52:26 +0100 Subject: [PATCH 01/22] port minimal 10.3 infra to lucene_snapshot --- .buildkite/pipelines/lucene-snapshot/build-snapshot.yml | 4 +++- docs/Versions.asciidoc | 4 ++-- .../src/main/java/org/elasticsearch/index/IndexVersions.java | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 8cf2a8aacbece..20a85c0593f31 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -2,7 +2,9 @@ steps: - trigger: apache-lucene-build-snapshot label: Trigger pipeline to build lucene snapshot key: lucene-build - if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "" + if: (build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "") + build: + branch: branch_10x - wait - label: Upload and update lucene snapshot command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 58195d7313a5a..aa02432b42b2a 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 10.2.0 -:lucene_version_path: 10_2_0 +:lucene_version: 10.3.0 +:lucene_version_path: 10_3_0 :jdk: 11.0.2 :jdk_major: 11 :build_type: tar diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 4c07be7d9200b..ebe0ca072c55e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -160,6 +160,8 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0); public static final IndexVersion USE_LUCENE101_POSTINGS_FORMAT = def(9_021_0_00, Version.LUCENE_10_1_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_2_0 = def(9_022_00_0, Version.LUCENE_10_2_0); + + public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ From 55c2ec7173b65693046cf67be25c0ddaf23adc8d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 20 Apr 2025 10:09:07 +0000 Subject: [PATCH 02/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-d18268d3d43 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 90b681c9636fe..84e52de94794b 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.2.0 +lucene = 10.3.0-snapshot-d18268d3d43 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index cbc22b35d7a35..dae3460520471 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2971,129 +2971,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 262a98b4af5408c86f3718667e66779d522f78d6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 21 Apr 2025 06:11:50 +0000 Subject: [PATCH 03/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-d18268d3d43 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index dae3460520471..cd512c0a4cfe4 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2973,127 +2973,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From 8ea85ee633b195ebf7f36b3df2300ec22594f7bd Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:36:33 +0100 Subject: [PATCH 04/22] Add getOffHeapByteSize to ES vector readers (#127104) --- .../index/codec/vectors/ES813FlatVectorFormat.java | 6 ++++++ .../codec/vectors/ES813Int8FlatVectorFormat.java | 6 ++++++ .../vectors/ES814ScalarQuantizedVectorsFormat.java | 6 ++++++ .../es816/ES816BinaryQuantizedVectorsReader.java | 14 ++++++++++++++ .../es818/ES818BinaryQuantizedVectorsReader.java | 14 ++++++++++++++ 5 files changed, 46 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index ab882c8b04648..325188624a2f4 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -31,6 +31,7 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -148,6 +149,11 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + @Override public void close() throws IOException { reader.close(); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index 662e4040511e2..4636a37d14f53 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -156,6 +157,11 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + @Override public void close() throws IOException { reader.close(); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 4c4fd00806954..204d9c6e9e630 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -38,6 +38,7 @@ import org.elasticsearch.simdvec.VectorSimilarityType; import java.io.IOException; +import java.util.Map; import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -227,6 +228,11 @@ public void close() throws IOException { public long ramBytesUsed() { return delegate.ramBytesUsed(); } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return delegate.getOffHeapByteSize(fieldInfo); + } } static final class ESFlatVectorsScorer implements FlatVectorsScorer { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java index fc20809ea7eed..57728e7f33254 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.codec.vectors.es816; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; import org.apache.lucene.index.ByteVectorValues; @@ -51,6 +52,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.elasticsearch.index.codec.vectors.es816.ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION; /** * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 @@ -253,6 +255,18 @@ public long ramBytesUsed() { return size; } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); + FieldEntry fe = fields.get(fieldInfo.name); + if (fe == null) { + assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; + return raw; + } + var quant = Map.of(VECTOR_DATA_EXTENSION, fe.vectorDataLength()); + return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); + } + public float[] getCentroid(String field) { FieldEntry fieldEntry = fields.get(field); if (fieldEntry != null) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java index 8036b8314cdc1..338c7668ef33d 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.codec.vectors.es818; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; import org.apache.lucene.index.ByteVectorValues; @@ -51,6 +52,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION; /** * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 @@ -252,6 +254,18 @@ public long ramBytesUsed() { return size; } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); + FieldEntry fe = fields.get(fieldInfo.name); + if (fe == null) { + assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; + return raw; + } + var quant = Map.of(VECTOR_DATA_EXTENSION, fe.vectorDataLength()); + return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); + } + public float[] getCentroid(String field) { FieldEntry fieldEntry = fields.get(field); if (fieldEntry != null) { From 4e88e93ccf3b204f12de528bdd3849f0533eac84 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 22 Apr 2025 06:12:52 +0000 Subject: [PATCH 05/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-b1978943b74 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 84e52de94794b..bd6fda14921fa 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-d18268d3d43 +lucene = 10.3.0-snapshot-b1978943b74 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index cd512c0a4cfe4..05f3f90f478dd 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2971,129 +2971,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 2a48a01b845982aae7cf4c3868d80c8a78f3e28d Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Tue, 22 Apr 2025 16:00:41 +0100 Subject: [PATCH 06/22] Fix index version parsing until 10.2.1 is released --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index d27a9f874401f..14c3a93758575 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -160,7 +160,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0); public static final IndexVersion USE_LUCENE101_POSTINGS_FORMAT = def(9_021_0_00, Version.LUCENE_10_1_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_2_0 = def(9_022_00_0, Version.LUCENE_10_2_0); - public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1); + public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, parseUnchecked("10.2.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0); /* From a266653397028993bbf90b7ea9856440c4ae3a70 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 23 Apr 2025 06:14:56 +0000 Subject: [PATCH 07/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-b1978943b74 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 05f3f90f478dd..a9c2d352755b4 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2973,127 +2973,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From 35c92c5d7a3e8b4925dc6e6ae395adc5f7e794cd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 24 Apr 2025 06:13:13 +0000 Subject: [PATCH 08/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-9920b1e9ddf --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index bd6fda14921fa..41cf300475d33 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-b1978943b74 +lucene = 10.3.0-snapshot-9920b1e9ddf bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index a9c2d352755b4..7a924dc5c30e3 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2971,129 +2971,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From c6a5edf127c667f1a5e717a13e5f397f77be14f5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 25 Apr 2025 06:13:27 +0000 Subject: [PATCH 09/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-bb3167e57c6 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 41cf300475d33..b37775b5b15c8 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-9920b1e9ddf +lucene = 10.3.0-snapshot-bb3167e57c6 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 7a924dc5c30e3..f48265dba4172 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2971,129 +2971,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 5cf19609ef8c0f770762ed1e1e01d9d7f4de141f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 26 Apr 2025 06:13:20 +0000 Subject: [PATCH 10/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-7b0ca79db30 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index b37775b5b15c8..850e688f9b76d 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-bb3167e57c6 +lucene = 10.3.0-snapshot-7b0ca79db30 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f48265dba4172..f5657f795487b 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2971,129 +2971,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 78b707c9a2c5f3b76e3a320db114a9c38e82f57c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 27 Apr 2025 06:15:15 +0000 Subject: [PATCH 11/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-7b0ca79db30 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f5657f795487b..43d2f3a4e9e4b 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2973,127 +2973,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From 88db358d120807201eee83af9f728d76cad45d96 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 28 Apr 2025 06:13:21 +0000 Subject: [PATCH 12/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-7b0ca79db30 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 43d2f3a4e9e4b..098cbb36ab1ff 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2973,127 +2973,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From 7aa7dd074771186e0f5d53ae54b99e652cfe58b7 Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Thu, 24 Apr 2025 17:16:33 +0100 Subject: [PATCH 13/22] Update off-heap dense vector stats to use Lucene's implementation. Much of the removals here are an effect revert of the reflective code put in place to get the stats internally prior to Lucene 10.3. --- .../server/cli/SystemJvmOptions.java | 7 - .../bootstrap/Elasticsearch.java | 7 +- .../codec/vectors/ES813FlatVectorFormat.java | 9 +- .../vectors/ES813Int8FlatVectorFormat.java | 9 +- .../ES814ScalarQuantizedVectorsFormat.java | 4 +- .../ES816BinaryQuantizedVectorsReader.java | 5 +- .../DirectIOLucene99FlatVectorsReader.java | 18 +- .../ES818BinaryQuantizedVectorsReader.java | 5 +- .../AssertingKnnVectorsReaderReflect.java | 83 ----- .../vectors/reflect/OffHeapByteSizeUtils.java | 92 ------ .../reflect/OffHeapReflectionUtils.java | 302 ------------------ .../codec/vectors/reflect/OffHeapStats.java | 23 -- .../elasticsearch/index/engine/Engine.java | 3 +- .../index/shard/DenseVectorStats.java | 4 +- .../elasticsearch/bootstrap/security.policy | 4 - .../vectors/ES813FlatVectorFormatTests.java | 3 +- .../ES813Int8FlatVectorFormatTests.java | 3 +- ...HnswScalarQuantizedVectorsFormatTests.java | 3 +- .../ES815BitFlatVectorFormatTests.java | 3 +- .../ES815HnswBitVectorsFormatTests.java | 3 +- ...S816BinaryQuantizedVectorsFormatTests.java | 3 +- ...HnswBinaryQuantizedVectorsFormatTests.java | 3 +- ...S818BinaryQuantizedVectorsFormatTests.java | 3 +- ...HnswBinaryQuantizedVectorsFormatTests.java | 3 +- 24 files changed, 34 insertions(+), 568 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 5191b60f1f8c9..93ee3850be04f 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -61,13 +61,6 @@ static List systemJvmOptions(Settings nodeSettings, final Map... classes) { for (final var clazz : classes) { try { - MethodHandles.lookup().ensureInitialized(clazz); + MethodHandles.publicLookup().ensureInitialized(clazz); } catch (IllegalAccessException unexpected) { throw new AssertionError(unexpected); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index 275443b989cfc..325188624a2f4 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -29,8 +29,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.Map; @@ -105,7 +103,7 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } - static class ES813FlatVectorReader extends KnnVectorsReader implements OffHeapStats { + static class ES813FlatVectorReader extends KnnVectorsReader { private final FlatVectorsReader reader; @@ -160,10 +158,5 @@ public Map getOffHeapByteSize(FieldInfo fieldInfo) { public void close() throws IOException { reader.close(); } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return OffHeapByteSizeUtils.getOffHeapByteSize(reader, fieldInfo); - } } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index 88df56b2a89d9..4636a37d14f53 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -27,8 +27,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.Map; @@ -113,7 +111,7 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } - public static class ES813FlatVectorReader extends KnnVectorsReader implements OffHeapStats { + public static class ES813FlatVectorReader extends KnnVectorsReader { private final FlatVectorsReader reader; @@ -168,10 +166,5 @@ public Map getOffHeapByteSize(FieldInfo fieldInfo) { public void close() throws IOException { reader.close(); } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return OffHeapByteSizeUtils.getOffHeapByteSize(reader, fieldInfo); - } } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 3f2baad122f27..204d9c6e9e630 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -34,8 +34,6 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues; import org.apache.lucene.util.quantization.QuantizedVectorsReader; import org.apache.lucene.util.quantization.ScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import org.elasticsearch.simdvec.VectorScorerFactory; import org.elasticsearch.simdvec.VectorSimilarityType; @@ -177,7 +175,7 @@ public long ramBytesUsed() { } } - static final class ES814ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader, OffHeapStats { + static final class ES814ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader { final Lucene99ScalarQuantizedVectorsReader delegate; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java index 7e492275928a8..5f290a72ba833 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java @@ -45,13 +45,10 @@ import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.elasticsearch.index.codec.vectors.BQVectorUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -61,7 +58,7 @@ * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @SuppressForbidden(reason = "Lucene classes") -public class ES816BinaryQuantizedVectorsReader extends FlatVectorsReader implements OffHeapStats { +public class ES816BinaryQuantizedVectorsReader extends FlatVectorsReader { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES816BinaryQuantizedVectorsReader.class); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java index 5f7dbb31a1ad4..d170d62ceb826 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Map; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -169,6 +170,12 @@ public long ramBytesUsed() { return SHALLOW_SIZE + fields.ramBytesUsed(); } + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + final FieldEntry entry = getFieldEntryOrThrow(fieldInfo.name); + return Map.of(DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength()); + } + @Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(vectorData); @@ -185,12 +192,17 @@ public FlatVectorsReader getMergeInstance() { } } - private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { + private FieldEntry getFieldEntryOrThrow(String field) { final FieldInfo info = fieldInfos.fieldInfo(field); - final FieldEntry fieldEntry; - if (info == null || (fieldEntry = fields.get(info.number)) == null) { + final FieldEntry entry; + if (info == null || (entry = fields.get(info.number)) == null) { throw new IllegalArgumentException("field=\"" + field + "\" not found"); } + return entry; + } + + private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { + final FieldEntry fieldEntry = getFieldEntryOrThrow(field); if (fieldEntry.vectorEncoding != expectedEncoding) { throw new IllegalArgumentException( "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + expectedEncoding diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java index 2ff7664e85e22..8e547f9b3fa20 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java @@ -46,13 +46,10 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.elasticsearch.index.codec.vectors.BQVectorUtils; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; @@ -62,7 +59,7 @@ * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @SuppressForbidden(reason = "Lucene classes") -public class ES818BinaryQuantizedVectorsReader extends FlatVectorsReader implements OffHeapStats { +public class ES818BinaryQuantizedVectorsReader extends FlatVectorsReader { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES818BinaryQuantizedVectorsReader.class); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java deleted file mode 100644 index bf47564c11b3a..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/AssertingKnnVectorsReaderReflect.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.codecs.KnnVectorsReader; -import org.elasticsearch.core.SuppressForbidden; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.security.AccessController; -import java.security.PrivilegedAction; - -/** - * Reflective access to unwrap non-accessible delegate in AssertingKnnVectorsReader. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class AssertingKnnVectorsReaderReflect { - - @SuppressForbidden(reason = "static type is not accessible") - public static KnnVectorsReader unwrapAssertingReader(KnnVectorsReader reader) { - try { - if (ASSERTING_ASSERT_KNN_READER_CLS != null && ASSERTING_ASSERT_KNN_READER_CLS.isAssignableFrom(reader.getClass())) { - return (KnnVectorsReader) GET_VECTOR_INDEX_LENGTH_HANDLE.invoke(reader); - } - } catch (Throwable t) { - handleThrowable(t); - } - return reader; - } - - private static final Class ASSERTING_ASSERT_KNN_READER_CLS = getAssertingReaderOrNull(); - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE = getDelegateFieldHandle(); - - private static Class getAssertingReaderOrNull() { - try { - return Class.forName("org.apache.lucene.tests.codecs.asserting.AssertingKnnVectorsFormat$AssertingKnnVectorsReader"); - } catch (ClassNotFoundException e) { - return null; - } - } - - private static MethodHandle getDelegateFieldHandle() { - try { - var cls = getAssertingReaderOrNull(); - if (cls == null) { - return MethodHandles.throwException(KnnVectorsReader.class, AssertionError.class); - } - var lookup = privilegedPrivateLookupIn(cls, MethodHandles.lookup()); - return lookup.findGetter(cls, "delegate", KnnVectorsReader.class); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } - - @SuppressWarnings("removal") - static MethodHandles.Lookup privilegedPrivateLookupIn(Class cls, MethodHandles.Lookup lookup) throws IllegalAccessException { - PrivilegedAction pa = () -> { - try { - return MethodHandles.privateLookupIn(cls, lookup); - } catch (IllegalAccessException e) { - throw new AssertionError("should not happen, check opens", e); - } - }; - return AccessController.doPrivileged(pa); - } - - static void handleThrowable(Throwable t) { - if (t instanceof Error error) { - throw error; - } else if (t instanceof RuntimeException runtimeException) { - throw runtimeException; - } else { - throw new AssertionError(t); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java deleted file mode 100644 index dcd9fc3b12737..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapByteSizeUtils.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; -import org.apache.lucene.index.FieldInfo; -import org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader; - -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Static utility methods to help retrieve desired off-heap vector index size. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class OffHeapByteSizeUtils { - - private OffHeapByteSizeUtils() {} // no instances - - public static Map getOffHeapByteSize(KnnVectorsReader reader, FieldInfo fieldInfo) { - reader = AssertingKnnVectorsReaderReflect.unwrapAssertingReader(reader); - switch (reader) { - case OffHeapStats offHeapStats -> { - return offHeapStats.getOffHeapByteSize(fieldInfo); - } - case Lucene99HnswVectorsReader hnswVectorsReader -> { - var graph = OffHeapReflectionUtils.getOffHeapByteSizeL99HNSW(hnswVectorsReader, fieldInfo); - var flat = getOffHeapByteSize(OffHeapReflectionUtils.getFlatVectorsReaderL99HNSW(hnswVectorsReader), fieldInfo); - return mergeOffHeapByteSizeMaps(graph, flat); - } - case Lucene99ScalarQuantizedVectorsReader scalarQuantizedVectorsReader -> { - var quant = OffHeapReflectionUtils.getOffHeapByteSizeSQ(scalarQuantizedVectorsReader, fieldInfo); - var raw = getOffHeapByteSize(OffHeapReflectionUtils.getFlatVectorsReaderSQ(scalarQuantizedVectorsReader), fieldInfo); - return mergeOffHeapByteSizeMaps(quant, raw); - } - case Lucene99FlatVectorsReader flatVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeF99FLT(flatVectorsReader, fieldInfo); - } - case DirectIOLucene99FlatVectorsReader flatVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeF99FLT(flatVectorsReader, fieldInfo); - } - case Lucene95HnswVectorsReader lucene95HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL95HNSW(lucene95HnswVectorsReader, fieldInfo); - } - case Lucene94HnswVectorsReader lucene94HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL94HNSW(lucene94HnswVectorsReader, fieldInfo); - } - case Lucene92HnswVectorsReader lucene92HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL92HNSW(lucene92HnswVectorsReader, fieldInfo); - } - case Lucene91HnswVectorsReader lucene91HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL91HNSW(lucene91HnswVectorsReader, fieldInfo); - } - case Lucene90HnswVectorsReader lucene90HnswVectorsReader -> { - return OffHeapReflectionUtils.getOffHeapByteSizeL90HNSW(lucene90HnswVectorsReader, fieldInfo); - } - case null, default -> { - assert false : "unexpected reader:" + reader; - } - } - return Map.of(); - } - - /** - * Merges the Maps returned by getOffHeapByteSize(FieldInfo). - * - *

This method is a convenience for aggregating the desired off-heap memory requirements for - * several fields. The keys in the returned map are a union of the keys in the given maps. Entries - * with the same key are summed. - */ - public static Map mergeOffHeapByteSizeMaps(Map map1, Map map2) { - return Stream.of(map1, map2) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java deleted file mode 100644 index 599a205508385..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapReflectionUtils.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader; -import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.VectorEncoding; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Map; - -import static java.lang.invoke.MethodType.methodType; - -/** - * Reflective access to non-accessible members of Lucene's KnnVectorsReader implementations. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public class OffHeapReflectionUtils { - - private OffHeapReflectionUtils() {} - - static final String FLAT_VECTOR_DATA_EXTENSION = "vec"; - static final String SQ_VECTOR_INDEX_EXTENSION = "veq"; - static final String HNSW_VECTOR_INDEX_EXTENSION = "vex"; - - private static final MethodHandle GET_FIELD_ENTRY_HNDL_SQ; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_SQ; - private static final VarHandle RAW_VECTORS_READER_HNDL_SQ; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L99FLT; - private static final MethodHandle VECTOR_DATA_LENGTH_HANDLE_L99FLT; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_DIOL99FLT; - private static final MethodHandle VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT; - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L99HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW; - private static final VarHandle FLAT_VECTORS_READER_HNDL_L99HNSW; - - static final Class L99_SQ_VR_CLS = Lucene99ScalarQuantizedVectorsReader.class; - static final Class L99_FLT_VR_CLS = Lucene99FlatVectorsReader.class; - static final Class DIOL99_FLT_VR_CLS = DirectIOLucene99FlatVectorsReader.class; - static final Class L99_HNSW_VR_CLS = Lucene99HnswVectorsReader.class; - - // old codecs - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L90HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L91HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L92HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L94HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW; - - private static final MethodHandle GET_FIELD_ENTRY_HANDLE_L95HNSW; - private static final MethodHandle GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW; - private static final MethodHandle GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW; - - static final Class L90_HNSW_VR_CLS = Lucene90HnswVectorsReader.class; - static final Class L91_HNSW_VR_CLS = Lucene91HnswVectorsReader.class; - static final Class L92_HNSW_VR_CLS = Lucene92HnswVectorsReader.class; - static final Class L94_HNSW_VR_CLS = Lucene94HnswVectorsReader.class; - static final Class L95_HNSW_VR_CLS = Lucene95HnswVectorsReader.class; - - static { - try { - // Lucene99ScalarQuantizedVectorsReader - var cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader$FieldEntry"); - var lookup = privilegedPrivateLookupIn(L99_SQ_VR_CLS, MethodHandles.lookup()); - var mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HNDL_SQ = lookup.findVirtual(L99_SQ_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_DATA_LENGTH_HANDLE_SQ = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - RAW_VECTORS_READER_HNDL_SQ = lookup.findVarHandle(L99_SQ_VR_CLS, "rawVectorsReader", FlatVectorsReader.class); - // Lucene99FlatVectorsReader - cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L99_FLT_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L99FLT = lookup.findVirtual(L99_FLT_VR_CLS, "getFieldEntry", mt); - VECTOR_DATA_LENGTH_HANDLE_L99FLT = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // DirectIOLucene99FlatVectorsReader - cls = Class.forName("org.elasticsearch.index.codec.vectors.es818.DirectIOLucene99FlatVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(DIOL99_FLT_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_DIOL99FLT = lookup.findVirtual(DIOL99_FLT_VR_CLS, "getFieldEntry", mt); - VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene99HnswVectorsReader - cls = Class.forName("org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L99_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L99HNSW = lookup.findVirtual(L99_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - lookup = privilegedPrivateLookupIn(L99_HNSW_VR_CLS, MethodHandles.lookup()); - FLAT_VECTORS_READER_HNDL_L99HNSW = lookup.findVarHandle(L99_HNSW_VR_CLS, "flatVectorsReader", FlatVectorsReader.class); - // Lucene90HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L90_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L90HNSW = lookup.findVirtual(L90_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW = lookup.findVirtual(cls, "indexDataLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene91HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L91_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L91HNSW = lookup.findVirtual(L91_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene92HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L92_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class); - GET_FIELD_ENTRY_HANDLE_L92HNSW = lookup.findVirtual(L92_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene94HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L94_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L94HNSW = lookup.findVirtual(L94_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - // Lucene95HnswVectorsReader - cls = Class.forName("org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsReader$FieldEntry"); - lookup = privilegedPrivateLookupIn(L95_HNSW_VR_CLS, MethodHandles.lookup()); - mt = methodType(cls, String.class, VectorEncoding.class); - GET_FIELD_ENTRY_HANDLE_L95HNSW = lookup.findVirtual(L95_HNSW_VR_CLS, "getFieldEntry", mt); - GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW = lookup.findVirtual(cls, "vectorIndexLength", methodType(long.class)); - GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW = lookup.findVirtual(cls, "vectorDataLength", methodType(long.class)); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeSQ(Lucene99ScalarQuantizedVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HNDL_SQ.invoke(reader, fieldInfo.name); - long len = (long) GET_VECTOR_DATA_LENGTH_HANDLE_SQ.invoke(entry); - return Map.of(SQ_VECTOR_INDEX_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - static FlatVectorsReader getFlatVectorsReaderSQ(Lucene99ScalarQuantizedVectorsReader reader) { - return (FlatVectorsReader) RAW_VECTORS_READER_HNDL_SQ.get(reader); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeF99FLT(Lucene99FlatVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L99FLT.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) VECTOR_DATA_LENGTH_HANDLE_L99FLT.invoke(entry); - return Map.of(FLAT_VECTOR_DATA_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeF99FLT(DirectIOLucene99FlatVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_DIOL99FLT.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) VECTOR_DATA_LENGTH_HANDLE_DIOL99FLT.invoke(entry); - return Map.of(FLAT_VECTOR_DATA_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL99HNSW(Lucene99HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L99HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long len = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L99HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, len); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - static FlatVectorsReader getFlatVectorsReaderL99HNSW(Lucene99HnswVectorsReader reader) { - return (FlatVectorsReader) FLAT_VECTORS_READER_HNDL_L99HNSW.get(reader); - } - - // old codecs - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL90HNSW(Lucene90HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L90HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L90HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L90HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL91HNSW(Lucene91HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L91HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L91HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L91HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL92HNSW(Lucene92HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L92HNSW.invoke(reader, fieldInfo.name); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L92HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L92HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL94HNSW(Lucene94HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L94HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L94HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L94HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressForbidden(reason = "static type is not accessible") - static Map getOffHeapByteSizeL95HNSW(Lucene95HnswVectorsReader reader, FieldInfo fieldInfo) { - try { - var entry = GET_FIELD_ENTRY_HANDLE_L95HNSW.invoke(reader, fieldInfo.name, fieldInfo.getVectorEncoding()); - long graph = (long) GET_VECTOR_INDEX_LENGTH_HANDLE_L95HNSW.invoke(entry); - long raw = (long) GET_VECTOR_DATA_LENGTH_HANDLE_L95HNSW.invoke(entry); - return Map.of(HNSW_VECTOR_INDEX_EXTENSION, graph, FLAT_VECTOR_DATA_EXTENSION, raw); - } catch (Throwable t) { - handleThrowable(t); - } - throw new AssertionError("should not reach here"); - } - - @SuppressWarnings("removal") - private static MethodHandles.Lookup privilegedPrivateLookupIn(Class cls, MethodHandles.Lookup lookup) { - PrivilegedAction pa = () -> { - try { - return MethodHandles.privateLookupIn(cls, lookup); - } catch (IllegalAccessException e) { - throw new AssertionError("should not happen, check opens", e); - } - }; - return AccessController.doPrivileged(pa); - } - - private static void handleThrowable(Throwable t) { - if (t instanceof Error error) { - throw error; - } else if (t instanceof RuntimeException runtimeException) { - throw runtimeException; - } else { - throw new AssertionError(t); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java deleted file mode 100644 index 79eb118f389cc..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/reflect/OffHeapStats.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.reflect; - -import org.apache.lucene.index.FieldInfo; - -import java.util.Map; - -/** - * Common interface to unify offHeapByteSize in ES' KnnVectorsReader implementations. - * Remove once KnnVectorsReaders::getOffHeapByteSize is available. - */ -public interface OffHeapStats { - - Map getOffHeapByteSize(FieldInfo fieldInfo); -} diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 6b3a4060cefad..91a39a4cc7bf8 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -62,7 +62,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.FieldInfosWithUsages; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.elasticsearch.index.mapper.DocumentParser; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; @@ -334,7 +333,7 @@ private DenseVectorStats getDenseVectorStats(final LeafReader atomicReader, List if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { vectorsReader = fieldsReader.getFieldReader(info.name); } - Map offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(vectorsReader, info); + Map offHeap = vectorsReader.getOffHeapByteSize(info); offHeapStats.put(info.name, offHeap); } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java b/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java index 6e74647899080..e60f325e2f5b1 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/DenseVectorStats.java @@ -9,11 +9,11 @@ package org.elasticsearch.index.shard; +import org.apache.lucene.codecs.KnnVectorsReader; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -92,7 +92,7 @@ public void add(DenseVectorStats other) { } else { this.offHeapStats = Stream.of(this.offHeapStats, other.offHeapStats) .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, OffHeapByteSizeUtils::mergeOffHeapByteSizeMaps)); + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, KnnVectorsReader::mergeOffHeapByteSizeMaps)); } } } diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 4f3bc1f92060b..55abdc84fc8fb 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -32,10 +32,6 @@ grant codeBase "${codebase.elasticsearch}" { // for plugin api dynamic settings instances permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect"; - - // For vector off-heap statistics, remove in Lucene 10.3 - permission java.lang.RuntimePermission "accessDeclaredMembers"; - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; //// Very special jar permissions: diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java index 80b2d5232ca65..9ea13750d8cd7 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; @@ -66,7 +65,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1, offHeap.size()); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java index 99c60c7bcc7f3..6a29bf08ae9a5 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; @@ -66,7 +65,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veq") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java index be715c8ab9175..b064321a70ae8 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java @@ -30,7 +30,6 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.nio.file.Path; @@ -199,7 +198,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java index b04e74adc53ae..c770a62479d38 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.junit.Before; import java.io.IOException; @@ -60,7 +59,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(1, offHeap.size()); assertTrue(offHeap.get("vec") > 0L); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java index 2e30fa0d16fa2..544dccc73c070 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import org.junit.Before; import java.io.IOException; @@ -60,7 +59,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertTrue(offHeap.get("vec") > 0L); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java index a18d86128e90f..6aec7b595ce8b 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java @@ -45,7 +45,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.index.codec.vectors.BQVectorUtils; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Locale; @@ -200,7 +199,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veb") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java index 5658935469302..a80240e228efc 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java @@ -41,7 +41,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.util.SameThreadExecutorService; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Arrays; @@ -147,7 +146,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java index 4e431cbe71eab..26bd905270f83 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.index.codec.vectors.BQVectorUtils; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Locale; @@ -200,7 +199,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(2, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertTrue(offHeap.get("veb") > 0L); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java index 63af621fafd31..cf7b710ef3093 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java @@ -41,7 +41,6 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.util.SameThreadExecutorService; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils; import java.io.IOException; import java.util.Arrays; @@ -151,7 +150,7 @@ public void testSimpleOffHeapSize() throws IOException { knnVectorsReader = fieldsReader.getFieldReader("f"); } var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertEquals(3, offHeap.size()); assertEquals(vector.length * Float.BYTES, (long) offHeap.get("vec")); assertEquals(1L, (long) offHeap.get("vex")); From b9d9f59e55dd1b973621c281fa50b99216789592 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Mon, 28 Apr 2025 18:47:46 +0100 Subject: [PATCH 14/22] Fix QueryPhaseTest that use trackTotalHits (#127470) This commit fixes a couple of test scenarios in QueryPhaseTest that incorrectly assert values around trackTotalHits. Test updated as per apache/lucene#14561 This test fix is needed when running with Lucene >= 10.2.1 --- .../elasticsearch/search/query/QueryPhaseTests.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 1f74668158e0e..0d7f16211aa51 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -55,6 +55,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; @@ -567,12 +568,13 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception { // size is lower than terminate_after context.setSize(5); // track_total_hits is lower than terminate_after - context.trackTotalHitsUpTo(randomIntBetween(1, 6)); + int trackTotalHits = randomIntBetween(1, 6); + context.trackTotalHitsUpTo(trackTotalHits); QueryPhase.executeQuery(context); // depending on docs distribution we may or may not be able to honor terminate_after: low scoring hits are skipped via // setMinCompetitiveScore, which bypasses terminate_after until the next leaf collector is pulled, when that happens. assertThat(context.queryResult().terminatedEarly(), either(is(true)).or(is(false))); - assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(7L)); + assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), greaterThanOrEqualTo((long) trackTotalHits)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(5)); } @@ -990,7 +992,10 @@ public void testMinScore() throws Exception { context.trackTotalHitsUpTo(5); QueryPhase.addCollectorsAndSearch(context); - assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value()); + TotalHits totalHits = context.queryResult().topDocs().topDocs.totalHits; + assertThat(totalHits.value(), greaterThanOrEqualTo(5L)); + var expectedRelation = totalHits.value() == 10 ? Relation.EQUAL_TO : Relation.GREATER_THAN_OR_EQUAL_TO; + assertThat(totalHits.relation(), is(expectedRelation)); } } From e211354827517d73caa42112eac0447733331ec0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 29 Apr 2025 06:13:33 +0000 Subject: [PATCH 15/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-4099fae40f5 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index ba8f83f81315c..1c2bf72fe81e4 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-7b0ca79db30 +lucene = 10.3.0-snapshot-4099fae40f5 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 5e3f725d2ad1e..fa39f08cb8d1d 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 124bf4b1293d8be2cabe713e0bb5c1536d9dde50 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 30 Apr 2025 06:12:34 +0000 Subject: [PATCH 16/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-85960129132 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 1c2bf72fe81e4..c5a3ed95e688b 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-4099fae40f5 +lucene = 10.3.0-snapshot-85960129132 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index fa39f08cb8d1d..ecd6b02b74411 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 4a098061751e89e67c2487df07d77a9ef97bf1e8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 1 May 2025 06:14:02 +0000 Subject: [PATCH 17/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-96553b7aff7 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index c5a3ed95e688b..2e36aadbf4c5e 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-85960129132 +lucene = 10.3.0-snapshot-96553b7aff7 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index ecd6b02b74411..32a555d36ac34 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 7429c2c40828bab67e5d67505532b91a8a68fa03 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 2 May 2025 06:12:50 +0000 Subject: [PATCH 18/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-9cf497e1667 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 2e36aadbf4c5e..c6c3fdbb68ebd 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-96553b7aff7 +lucene = 10.3.0-snapshot-9cf497e1667 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 32a555d36ac34..26de34e979fa9 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From 7bbecb6cba2046e021a5f620fbef7e4b59eb7ce7 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 2 May 2025 08:44:04 +0100 Subject: [PATCH 19/22] Use a hint to specify DirectIO to HybridDirectory (#127601) Remove the forked DirectIO lucene format --- .../elasticsearch/index/IndexVersions.java | 2 +- ...exInputSupplier.java => DirectIOHint.java} | 11 +- .../DirectIOLucene99FlatVectorsFormat.java | 68 +++- .../DirectIOLucene99FlatVectorsReader.java | 358 ------------------ .../ES818BinaryQuantizedVectorsFormat.java | 9 +- .../vectors/es818/MergeReaderWrapper.java | 21 +- .../index/store/FsDirectoryFactory.java | 22 +- .../input/CachedBlobContainerIndexInput.java | 41 +- 8 files changed, 127 insertions(+), 405 deletions(-) rename server/src/main/java/org/elasticsearch/index/codec/vectors/es818/{DirectIOIndexInputSupplier.java => DirectIOHint.java} (59%) delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 5843c562dd5ff..b496e6d861ca6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -163,7 +163,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1); public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1); public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1); - + public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java similarity index 59% rename from server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java rename to server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java index 0640a5dacce65..73b7182911114 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java @@ -10,14 +10,7 @@ package org.elasticsearch.index.codec.vectors.es818; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import java.io.IOException; - -/** - * A hook for {@link DirectIOLucene99FlatVectorsReader} to specify the input should be opened using DirectIO. - * Remove when IOContext allows more extensible payloads to be specified. - */ -public interface DirectIOIndexInputSupplier { - IndexInput openInputDirect(String name, IOContext context) throws IOException; +public enum DirectIOHint implements IOContext.FileOpenHint { + INSTANCE } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java index 02de4d5450d35..a0cd6dbf65688 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java @@ -27,8 +27,14 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.FlushInfo; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.ReadAdvice; import java.io.IOException; +import java.util.Optional; +import java.util.Set; /** * Copied from Lucene99FlatVectorsFormat in Lucene 10.1 @@ -61,22 +67,62 @@ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOExceptio return new Lucene99FlatVectorsWriter(state, vectorsScorer); } + private static final IOContext DIRECT_IO_CONTEXT = new IOContext() { + @Override + public Context context() { + return Context.DEFAULT; + } + + @Override + public MergeInfo mergeInfo() { + return null; + } + + @Override + public FlushInfo flushInfo() { + return null; + } + + @Override + public Set hints() { + return Set.of(DirectIOHint.INSTANCE); + } + + @Override + public IOContext withHints(FileOpenHint... hints) { + return this; + } + + @Override + public Optional readAdvice() { + return Optional.empty(); + } + + @Override + public IOContext withReadAdvice(ReadAdvice advice) { + return this; + } + }; + @Override public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { - if (DirectIOLucene99FlatVectorsReader.shouldUseDirectIO(state)) { - // Use mmap for merges and direct I/O for searches. - // TODO: Open the mmap file with sequential access instead of random (current behavior). - return new MergeReaderWrapper( - new DirectIOLucene99FlatVectorsReader(state, vectorsScorer), - new Lucene99FlatVectorsReader(state, vectorsScorer) - ); - } else { - return new Lucene99FlatVectorsReader(state, vectorsScorer); - } + SegmentReadState directIOState = new SegmentReadState( + state.directory, + state.segmentInfo, + state.fieldInfos, + DIRECT_IO_CONTEXT, + state.segmentSuffix + ); + // Use mmap for merges and direct I/O for searches. + // TODO: Open the mmap file with sequential access instead of random (current behavior). + return new MergeReaderWrapper( + new Lucene99FlatVectorsReader(directIOState, vectorsScorer), + new Lucene99FlatVectorsReader(state, vectorsScorer) + ); } @Override public String toString() { - return "ES818FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')'; + return "Lucene99FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')'; } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java deleted file mode 100644 index 0d087cb5d73c9..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java +++ /dev/null @@ -1,358 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2025 Elasticsearch B.V. - */ -package org.elasticsearch.index.codec.vectors.es818; - -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; -import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues; -import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues; -import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.internal.hppc.IntObjectHashMap; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.ReadAdvice; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.SuppressForbidden; -import org.apache.lucene.util.hnsw.RandomVectorScorer; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; - -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; - -/** Copied from Lucene99FlatVectorsReader in Lucene 10.2, then modified to support DirectIOIndexInputSupplier */ -@SuppressForbidden(reason = "Copied from lucene") -public class DirectIOLucene99FlatVectorsReader extends FlatVectorsReader { - - private static final boolean USE_DIRECT_IO = Boolean.parseBoolean(System.getProperty("vector.rescoring.directio", "true")); - - private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(DirectIOLucene99FlatVectorsReader.class); - - private final IntObjectHashMap fields = new IntObjectHashMap<>(); - private final IndexInput vectorData; - private final FieldInfos fieldInfos; - - public DirectIOLucene99FlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer) throws IOException { - super(scorer); - int versionMeta = readMetadata(state); - this.fieldInfos = state.fieldInfos; - boolean success = false; - try { - vectorData = openDataInput( - state, - versionMeta, - DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, - DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_CODEC_NAME, - // Flat formats are used to randomly access vectors from their node ID that is stored - // in the HNSW graph. - state.context.withReadAdvice(ReadAdvice.RANDOM) - ); - success = true; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(this); - } - } - } - - public static boolean shouldUseDirectIO(SegmentReadState state) { - return USE_DIRECT_IO && state.directory instanceof DirectIOIndexInputSupplier; - } - - private int readMetadata(SegmentReadState state) throws IOException { - String metaFileName = IndexFileNames.segmentFileName( - state.segmentInfo.name, - state.segmentSuffix, - DirectIOLucene99FlatVectorsFormat.META_EXTENSION - ); - int versionMeta = -1; - try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { - Throwable priorE = null; - try { - versionMeta = CodecUtil.checkIndexHeader( - meta, - DirectIOLucene99FlatVectorsFormat.META_CODEC_NAME, - DirectIOLucene99FlatVectorsFormat.VERSION_START, - DirectIOLucene99FlatVectorsFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - readFields(meta, state.fieldInfos); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(meta, priorE); - } - } - return versionMeta; - } - - private static IndexInput openDataInput( - SegmentReadState state, - int versionMeta, - String fileExtension, - String codecName, - IOContext context - ) throws IOException { - String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); - // use direct IO for accessing raw vector data for searches - IndexInput in = USE_DIRECT_IO && state.directory instanceof DirectIOIndexInputSupplier did - ? did.openInputDirect(fileName, context) - : state.directory.openInput(fileName, context); - boolean success = false; - try { - int versionVectorData = CodecUtil.checkIndexHeader( - in, - codecName, - DirectIOLucene99FlatVectorsFormat.VERSION_START, - DirectIOLucene99FlatVectorsFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - if (versionMeta != versionVectorData) { - throw new CorruptIndexException( - "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, - in - ); - } - CodecUtil.retrieveChecksum(in); - success = true; - return in; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(in); - } - } - } - - private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException { - for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { - FieldInfo info = infos.fieldInfo(fieldNumber); - if (info == null) { - throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); - } - FieldEntry fieldEntry = FieldEntry.create(meta, info); - fields.put(info.number, fieldEntry); - } - } - - @Override - public long ramBytesUsed() { - return SHALLOW_SIZE + fields.ramBytesUsed(); - } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - final FieldEntry entry = getFieldEntryOrThrow(fieldInfo.name); - return Map.of(DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength()); - } - - @Override - public void checkIntegrity() throws IOException { - CodecUtil.checksumEntireFile(vectorData); - } - - @Override - public FlatVectorsReader getMergeInstance() { - try { - // Update the read advice since vectors are guaranteed to be accessed sequentially for merge - this.vectorData.updateReadAdvice(ReadAdvice.SEQUENTIAL); - return this; - } catch (IOException exception) { - throw new UncheckedIOException(exception); - } - } - - private FieldEntry getFieldEntryOrThrow(String field) { - final FieldInfo info = fieldInfos.fieldInfo(field); - final FieldEntry entry; - if (info == null || (entry = fields.get(info.number)) == null) { - throw new IllegalArgumentException("field=\"" + field + "\" not found"); - } - return entry; - } - - private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { - final FieldEntry fieldEntry = getFieldEntryOrThrow(field); - if (fieldEntry.vectorEncoding != expectedEncoding) { - throw new IllegalArgumentException( - "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + expectedEncoding - ); - } - return fieldEntry; - } - - @Override - public FloatVectorValues getFloatVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); - return OffHeapFloatVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ); - } - - @Override - public ByteVectorValues getByteVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.BYTE); - return OffHeapByteVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); - return vectorScorer.getRandomVectorScorer( - fieldEntry.similarityFunction, - OffHeapFloatVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ), - target - ); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.BYTE); - return vectorScorer.getRandomVectorScorer( - fieldEntry.similarityFunction, - OffHeapByteVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ), - target - ); - } - - @Override - public void finishMerge() throws IOException { - // This makes sure that the access pattern hint is reverted back since HNSW implementation - // needs it - this.vectorData.updateReadAdvice(ReadAdvice.RANDOM); - } - - @Override - public void close() throws IOException { - IOUtils.close(vectorData); - } - - private record FieldEntry( - VectorSimilarityFunction similarityFunction, - VectorEncoding vectorEncoding, - long vectorDataOffset, - long vectorDataLength, - int dimension, - int size, - OrdToDocDISIReaderConfiguration ordToDoc, - FieldInfo info - ) { - - FieldEntry { - if (similarityFunction != info.getVectorSimilarityFunction()) { - throw new IllegalStateException( - "Inconsistent vector similarity function for field=\"" - + info.name - + "\"; " - + similarityFunction - + " != " - + info.getVectorSimilarityFunction() - ); - } - int infoVectorDimension = info.getVectorDimension(); - if (infoVectorDimension != dimension) { - throw new IllegalStateException( - "Inconsistent vector dimension for field=\"" + info.name + "\"; " + infoVectorDimension + " != " + dimension - ); - } - - int byteSize = switch (info.getVectorEncoding()) { - case BYTE -> Byte.BYTES; - case FLOAT32 -> Float.BYTES; - }; - long vectorBytes = Math.multiplyExact((long) infoVectorDimension, byteSize); - long numBytes = Math.multiplyExact(vectorBytes, size); - if (numBytes != vectorDataLength) { - throw new IllegalStateException( - "Vector data length " - + vectorDataLength - + " not matching size=" - + size - + " * dim=" - + dimension - + " * byteSize=" - + byteSize - + " = " - + numBytes - ); - } - } - - static FieldEntry create(IndexInput input, FieldInfo info) throws IOException { - final VectorEncoding vectorEncoding = readVectorEncoding(input); - final VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); - final var vectorDataOffset = input.readVLong(); - final var vectorDataLength = input.readVLong(); - final var dimension = input.readVInt(); - final var size = input.readInt(); - final var ordToDoc = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); - return new FieldEntry(similarityFunction, vectorEncoding, vectorDataOffset, vectorDataLength, dimension, size, ordToDoc, info); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java index c3e12a765eb5f..3ab49ab590346 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; @@ -86,6 +87,8 @@ */ public class ES818BinaryQuantizedVectorsFormat extends FlatVectorsFormat { + private static final boolean USE_DIRECT_IO = Boolean.parseBoolean(System.getProperty("vector.rescoring.directio", "true")); + public static final String BINARIZED_VECTOR_COMPONENT = "BVEC"; public static final String NAME = "ES818BinaryQuantizedVectorsFormat"; @@ -97,9 +100,9 @@ public class ES818BinaryQuantizedVectorsFormat extends FlatVectorsFormat { static final String VECTOR_DATA_EXTENSION = "veb"; static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16; - private static final DirectIOLucene99FlatVectorsFormat rawVectorFormat = new DirectIOLucene99FlatVectorsFormat( - FlatVectorScorerUtil.getLucene99FlatVectorsScorer() - ); + private static final FlatVectorsFormat rawVectorFormat = USE_DIRECT_IO + ? new DirectIOLucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()) + : new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()); private static final ES818BinaryFlatVectorsScorer scorer = new ES818BinaryFlatVectorsScorer( FlatVectorScorerUtil.getLucene99FlatVectorsScorer() diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java index eaafbe7bfc947..eef9f5cc28b07 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Accountable; @@ -20,6 +21,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Map; class MergeReaderWrapper extends FlatVectorsReader { @@ -57,6 +59,16 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { return mainReader.getByteVectorValues(field); } + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + mainReader.search(field, target, knnCollector, acceptDocs); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + mainReader.search(field, target, knnCollector, acceptDocs); + } + @Override public FlatVectorsReader getMergeInstance() { return mergeReader; @@ -73,13 +85,8 @@ public Collection getChildResources() { } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - mainReader.search(field, target, knnCollector, acceptDocs); - } - - @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - mainReader.search(field, target, knnCollector, acceptDocs); + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return mainReader.getOffHeapByteSize(fieldInfo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 76dfee2f7f2de..bc1d469ab3486 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -26,7 +26,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.codec.vectors.es818.DirectIOIndexInputSupplier; +import org.elasticsearch.index.codec.vectors.es818.DirectIOHint; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -116,7 +116,7 @@ public static boolean isHybridFs(Directory directory) { return unwrap instanceof HybridDirectory; } - static final class HybridDirectory extends NIOFSDirectory implements DirectIOIndexInputSupplier { + static final class HybridDirectory extends NIOFSDirectory { private final MMapDirectory delegate; private final DirectIODirectory directIODelegate; @@ -143,7 +143,11 @@ protected boolean useDirectIO(String name, IOContext context, OptionalLong fileL @Override public IndexInput openInput(String name, IOContext context) throws IOException { - if (useDelegate(name, context)) { + if (directIODelegate != null && context.hints().contains(DirectIOHint.INSTANCE)) { + ensureOpen(); + ensureCanRead(name); + return directIODelegate.openInput(name, context); + } else if (useDelegate(name, context)) { // we need to do these checks on the outer directory since the inner doesn't know about pending deletes ensureOpen(); ensureCanRead(name); @@ -159,18 +163,6 @@ public IndexInput openInput(String name, IOContext context) throws IOException { } } - @Override - public IndexInput openInputDirect(String name, IOContext context) throws IOException { - if (directIODelegate == null) { - return openInput(name, context); - } - // we need to do these checks on the outer directory since the inner doesn't know about pending deletes - ensureOpen(); - ensureCanRead(name); - - return directIODelegate.openInput(name, context); - } - @Override public void close() throws IOException { IOUtils.close(super::close, delegate); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java index 4711043fff281..e54fe892ecaab 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java @@ -9,7 +9,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MergeInfo; import org.apache.lucene.store.ReadAdvice; import org.elasticsearch.blobcache.BlobCacheUtils; import org.elasticsearch.blobcache.common.ByteRange; @@ -21,6 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; import java.util.function.Supplier; @@ -36,7 +40,42 @@ public class CachedBlobContainerIndexInput extends MetadataCachingIndexInput { * a complete part of the {@link #fileInfo} at once in the cache and should not be * used for anything else than what the {@link #prefetchPart(int, Supplier)} method does. */ - public static final IOContext CACHE_WARMING_CONTEXT = new IOContext(IOContext.Context.DEFAULT, null, null, ReadAdvice.NORMAL); + public static final IOContext CACHE_WARMING_CONTEXT = new IOContext() { + @Override + public Context context() { + return Context.DEFAULT; + } + + @Override + public MergeInfo mergeInfo() { + return null; + } + + @Override + public FlushInfo flushInfo() { + return null; + } + + @Override + public Set hints() { + return Set.of(); + } + + @Override + public IOContext withHints(FileOpenHint... hints) { + return this; + } + + @Override + public Optional readAdvice() { + return Optional.of(ReadAdvice.NORMAL); + } + + @Override + public IOContext withReadAdvice(ReadAdvice advice) { + return this; + } + }; private static final Logger logger = LogManager.getLogger(CachedBlobContainerIndexInput.class); From bc65772b33e075f31e0eb78b37c6eac5bfd8573f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 3 May 2025 06:13:32 +0000 Subject: [PATCH 20/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-40c3848eb29 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index c6c3fdbb68ebd..abfb0310f6a44 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-9cf497e1667 +lucene = 10.3.0-snapshot-40c3848eb29 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index a12bd99035937..ebb3b8de769c0 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + From bd4edfa67232fc4ad33cff6a7828cd810db8af36 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 4 May 2025 06:13:07 +0000 Subject: [PATCH 21/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-40c3848eb29 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index ebb3b8de769c0..655224960e023 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2953,127 +2953,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From 7423500ff7a8644ab1f475bdb51974a134d09ef0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 5 May 2025 06:14:17 +0000 Subject: [PATCH 22/22] [Automated] Update Lucene snapshot to 10.3.0-snapshot-c50f5ee3d75 --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 150 ++++++++++++------------ 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index abfb0310f6a44..c22d3d93533bf 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.1.0 -lucene = 10.3.0-snapshot-40c3848eb29 +lucene = 10.3.0-snapshot-c50f5ee3d75 bundled_jdk_vendor = openjdk bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 655224960e023..ac4e2799488da 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2951,129 +2951,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + +