From db3eb3a896da67d02266710b750cd45625961aff Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 5 Mar 2025 11:59:27 -0800 Subject: [PATCH 1/6] wip --- .../mapper/extras/ScaledFloatFieldMapper.java | 2 +- .../mapper/AbstractGeometryFieldMapper.java | 73 +++++++++- .../index/mapper/BooleanFieldMapper.java | 6 +- .../index/mapper/DateFieldMapper.java | 2 +- .../FallbackSyntheticSourceBlockLoader.java | 28 ++-- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 3 +- .../index/mapper/BlockLoaderTestCase.java | 11 +- .../NumberFieldBlockLoaderTestCase.java | 2 +- .../datageneration/DocumentGenerator.java | 8 +- .../logsdb/datageneration/FieldType.java | 21 ++- .../datageneration/MappingGenerator.java | 2 +- .../logsdb/datageneration/Template.java | 2 +- .../datageneration/TemplateGenerator.java | 2 +- .../datageneration/datasource/DataSource.java | 1 + .../datasource/DataSourceHandler.java | 8 ++ .../datasource/DataSourceRequest.java | 17 ++- .../datasource/DataSourceResponse.java | 5 + .../DefaultFieldDataGeneratorHandler.java | 23 ++++ .../DefaultMappingParametersHandler.java | 10 +- .../matchers/source/FieldSpecificMatcher.java | 40 ++++++ .../matchers/source/SourceMatcher.java | 4 + .../logsdb/datageneration/FieldTypeTests.java | 20 +++ .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- .../GeoShapeWithDocValuesFieldMapper.java | 15 ++- .../index/mapper/ShapeFieldMapper.java | 16 ++- .../GeoShapeDataSourceHandler.java | 73 ++++++++++ .../GeoShapeFieldDataGenerator.java | 51 +++++++ .../mapper/GeoShapeFieldBlockLoaderTests.java | 126 ++++++++++++++++++ ...GeoShapeWithDocValuesFieldMapperTests.java | 2 +- .../index/mapper/ShapeFieldMapperTests.java | 2 +- .../spatial/ingest/CircleProcessorTests.java | 1 + .../geogrid/GeoShapeGeoGridTestCase.java | 1 + .../CartesianShapeBoundsAggregatorTests.java | 5 + ...CartesianShapeCentroidAggregatorTests.java | 14 +- .../GeoShapeBoundsAggregatorTests.java | 5 + .../GeoShapeCentroidAggregatorTests.java | 5 + 37 files changed, 567 insertions(+), 43 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java create mode 100644 test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index e229c54b195ab..70622f72d1167 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -343,7 +343,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { if (coerce && value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 6e00cc765bd8b..8fc91c435cb97 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryFormatterFactory; @@ -67,12 +68,16 @@ public abstract void parse(XContentParser parser, CheckedConsumer consumer) { try (XContentParser parser = wrapObject(sourceMap)) { - parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE); + parseFromSource(parser, consumer); } catch (IOException e) { throw new UncheckedIOException(e); } } + private void parseFromSource(XContentParser parser, Consumer consumer) throws IOException { + parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE); + } + /** * Normalize a geometry when reading from source. When reading from source we can skip * some expensive steps as the geometry has already been indexed. @@ -187,6 +192,72 @@ protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { } protected abstract Object nullValueAsSource(T nullValue); + + protected BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { + Function, List> formatter = getFormatter(GeometryFormatterFactory.WKB); + + return new FallbackSyntheticSourceBlockLoader(new GeometriesFallbackSyntheticSourceReader(geometryParser, formatter), name()) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + }; + } + + private class GeometriesFallbackSyntheticSourceReader implements FallbackSyntheticSourceBlockLoader.Reader { + private final Parser geometryParser; + private final Function, List> formatter; + + private GeometriesFallbackSyntheticSourceReader(Parser geometryParser, Function, List> formatter) { + this.geometryParser = geometryParser; + this.formatter = formatter; + } + + @Override + public void convertValue(Object value, List accumulator) { + final List values = new ArrayList<>(); + + geometryParser.fetchFromSource(value, values::add); + var formatted = formatter.apply(values); + + for (var formattedValue : formatted) { + if (formattedValue instanceof byte[] wkb) { + accumulator.add(new BytesRef(wkb)); + } else { + throw new IllegalArgumentException( + "Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName() + ); + } + } + } + + @Override + public void parse(XContentParser parser, List accumulator) throws IOException { + final List values = new ArrayList<>(); + + geometryParser.parseFromSource(parser, values::add); + var formatted = formatter.apply(values); + + for (var formattedValue : formatted) { + if (formattedValue instanceof byte[] wkb) { + accumulator.add(new BytesRef(wkb)); + } else { + throw new IllegalArgumentException( + "Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName() + ); + } + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder; + + for (var value : values) { + bytesRefBuilder.appendBytesRef(value); + } + } + } } private final Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index d7f4a9d4241a3..975c9d84740d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -331,7 +331,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { try { @@ -360,10 +360,10 @@ protected void parseNonNullValue(XContentParser parser, List accumulato @Override public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { - var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder; + var booleanBuilder = (BlockLoader.BooleanBuilder) blockBuilder; for (var value : values) { - longBuilder.appendBoolean(value); + booleanBuilder.appendBoolean(value); } } }; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 72eb8841d5bae..d8019c058b509 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -963,7 +963,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { Function dateParser = this::parse; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index 8474b754e951d..f606de4691e53 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -235,13 +235,6 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue, } private void parseWithReader(XContentParser parser, List blockValues) throws IOException { - if (parser.currentToken() == XContentParser.Token.START_ARRAY) { - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - reader.parse(parser, blockValues); - } - return; - } - reader.parse(parser, blockValues); } @@ -274,10 +267,15 @@ public interface Reader { void writeToBlock(List values, Builder blockBuilder); } - public abstract static class ReaderWithNullValueSupport implements Reader { + /** + * Reader for field types that don't parse arrays (arrays are always treated as multiple values) + * as opposed to field types that treat arrays as special cases (for example point). + * @param + */ + public abstract static class SingleValueReader implements Reader { private final Object nullValue; - public ReaderWithNullValueSupport(Object nullValue) { + public SingleValueReader(Object nullValue) { this.nullValue = nullValue; } @@ -289,6 +287,18 @@ public void parse(XContentParser parser, List accumulator) throws IOException } return; } + if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + if (nullValue != null) { + convertValue(nullValue, accumulator); + } + } else { + parseNonNullValue(parser, accumulator); + } + } + return; + } parseNonNullValue(parser, accumulator); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e698920433981..3e7f595736145 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -770,7 +770,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueBytes) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueBytes) { @Override public void convertValue(Object value, List accumulator) { String stringValue = ((BytesRef) value).utf8ToString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7c644fd432107..f2768a842f0c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1729,8 +1729,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport< - Number> { + abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader { private final NumberType type; private final Number nullValue; private final boolean coerce; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index 90c4bd38b39ff..9b797855515e7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.DocumentGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.Mapping; import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; @@ -35,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,14 +59,18 @@ public static List args() { public record Params(boolean syntheticSource, MappedFieldType.FieldExtractPreference preference) {} - private final FieldType fieldType; + private final String fieldType; protected final Params params; private final String fieldName; private final MappingGenerator mappingGenerator; private final DocumentGenerator documentGenerator; - protected BlockLoaderTestCase(FieldType fieldType, Params params) { + protected BlockLoaderTestCase(String fieldType, Params params) { + this(fieldType, List.of(), params); + } + + protected BlockLoaderTestCase(String fieldType, Collection customHandlers, Params params) { this.fieldType = fieldType; this.params = params; this.fieldName = randomAlphaOfLengthBetween(5, 10); @@ -87,6 +91,7 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle( return new DataSourceResponse.ObjectMappingParametersGenerator(HashMap::new); // just defaults } })) + .withDataSourceHandlers(customHandlers) .build(); this.mappingGenerator = new MappingGenerator(specification); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java index 2902439cb5e57..3be481452f388 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java @@ -17,7 +17,7 @@ public abstract class NumberFieldBlockLoaderTestCase extends BlockLoaderTestCase { public NumberFieldBlockLoaderTestCase(FieldType fieldType, Params params) { - super(fieldType, params); + super(fieldType.toString(), params); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java index cfdec40bf9190..67c6c7092b3b2 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java @@ -61,12 +61,14 @@ private void generateFields(Map document, Map arrayLength = objectArrayGenerator.lengthGenerator().get(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 851812268e9ba..5f3c6a959896a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -25,7 +25,7 @@ import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator; /** - * Lists all leaf field types that are supported for data generation. + * Lists all leaf field types that are supported for data generation by default. */ public enum FieldType { KEYWORD("keyword"), @@ -66,6 +66,25 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { }; } + public static FieldType tryParse(String name) { + return switch (name) { + case "keyword" -> FieldType.KEYWORD; + case "long" -> FieldType.LONG; + case "unsigned_long" -> FieldType.UNSIGNED_LONG; + case "integer" -> FieldType.INTEGER; + case "short" -> FieldType.SHORT; + case "byte" -> FieldType.BYTE; + case "double" -> FieldType.DOUBLE; + case "float" -> FieldType.FLOAT; + case "half_float" -> FieldType.HALF_FLOAT; + case "scaled_float" -> FieldType.SCALED_FLOAT; + case "counted_keyword" -> FieldType.COUNTED_KEYWORD; + case "boolean" -> FieldType.BOOLEAN; + case "date" -> FieldType.DATE; + default -> null; + }; + } + @Override public String toString() { return name; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java index 9a7add456fd67..da26e9957198d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java @@ -116,7 +116,7 @@ private void generateMapping( mappingParameters.putAll(mappingParametersGenerator.get()); // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. - if (leaf.type() == FieldType.KEYWORD) { + if (leaf.type().equals(FieldType.KEYWORD.toString())) { context.addCopyToCandidate(fieldName); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java index a46e07446052a..71c8a54e5e0f6 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java @@ -21,7 +21,7 @@ public record Template(Map template) { public sealed interface Entry permits Leaf, Object {} - public record Leaf(String name, FieldType type) implements Entry {} + public record Leaf(String name, String type) implements Entry {} public record Object(String name, boolean nested, Map children) implements Entry {} } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java index f0a3a866a2673..db6abe655caa8 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java @@ -59,7 +59,7 @@ private void generateChildFields(Map mapping, int depth, generateChildFields(children, depth + 1, nestedFieldsCount); } else { var fieldTypeInfo = fieldTypeGenerator.get(); - mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType())); + mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType().toString())); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java index 2fe62d2c967df..cd0b9184ce42c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java @@ -32,6 +32,7 @@ public DataSource(Collection additionalHandlers) { this.handlers.addAll(additionalHandlers); + this.handlers.add(new DefaultFieldDataGeneratorHandler()); this.handlers.add(new DefaultPrimitiveTypesHandler()); this.handlers.add(new DefaultWrappersHandler()); this.handlers.add(new DefaultObjectGenerationHandler()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 2a17f9311faa9..829d2ae0d24a9 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -10,6 +10,10 @@ package org.elasticsearch.logsdb.datageneration.datasource; public interface DataSourceHandler { + default DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + return null; + } + default DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator request) { return null; } @@ -54,6 +58,10 @@ default DataSourceResponse.InstantGenerator handle(DataSourceRequest.InstantGene return null; } + default DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGenerator request) { + return null; + } + default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 2a1ca7297d7db..47787fb6abc0c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -11,7 +11,6 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import java.util.Set; @@ -21,6 +20,14 @@ public interface DataSourceRequest { TResponse accept(DataSourceHandler handler); + record FieldDataGenerator(String fieldName, String fieldType, DataSource dataSource) + implements + DataSourceRequest { + public DataSourceResponse.FieldDataGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record LongGenerator() implements DataSourceRequest { public DataSourceResponse.LongGenerator accept(DataSourceHandler handler) { return handler.handle(this); @@ -87,6 +94,12 @@ public DataSourceResponse.InstantGenerator accept(DataSourceHandler handler) { } } + record GeoShapeGenerator() implements DataSourceRequest { + public DataSourceResponse.GeoShapeGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record NullWrapper() implements DataSourceRequest { public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) { return handler.handle(this); @@ -141,7 +154,7 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) record LeafMappingParametersGenerator( String fieldName, - FieldType fieldType, + String fieldType, Set eligibleCopyToFields, DynamicMapping dynamicMapping ) implements DataSourceRequest { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index e7a64471e024c..bbe98ccc8e5ca 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -9,6 +9,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.logsdb.datageneration.FieldType; import java.time.Instant; @@ -18,6 +19,8 @@ import java.util.function.Supplier; public interface DataSourceResponse { + record FieldDataGenerator(org.elasticsearch.logsdb.datageneration.FieldDataGenerator generator) implements DataSourceResponse {} + record LongGenerator(Supplier generator) implements DataSourceResponse {} record UnsignedLongGenerator(Supplier generator) implements DataSourceResponse {} @@ -40,6 +43,8 @@ record BooleanGenerator(Supplier generator) implements DataSourceRespon record InstantGenerator(Supplier generator) implements DataSourceResponse {} + record GeoShapeGenerator(Supplier generator) implements DataSourceResponse {} + record NullWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java new file mode 100644 index 0000000000000..62578ec08cbcf --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.datasource; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class DefaultFieldDataGeneratorHandler implements DataSourceHandler { + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + var fieldType = FieldType.tryParse(request.fieldType()); + if (fieldType == null) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(fieldType.generator(request.fieldName(), request.dataSource())); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 93faf795ff565..369f26d36b54d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -27,6 +27,12 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + var fieldType = FieldType.tryParse(request.fieldType()); + if (fieldType == null) { + // This is a custom field type + return null; + } + var map = new HashMap(); map.put("store", ESTestCase.randomBoolean()); map.put("index", ESTestCase.randomBoolean()); @@ -35,9 +41,9 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques map.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); } - return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) { + return new DataSourceResponse.LeafMappingParametersGenerator(switch (fieldType) { case KEYWORD -> keywordMapping(request, map); - case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, request.fieldType()); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, fieldType); case SCALED_FLOAT -> scaledFloatMapping(map); case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); case BOOLEAN -> booleanMapping(map); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index f424a7ecf45ed..1448019d64a8a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -473,6 +473,46 @@ Object convert(Object value, Object nullValue, DateTimeFormatter dateTimeFormatt } } + class GeoShapeMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + GeoShapeMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + // Since fallback synthetic source is used, should match exactly + return actual.equals(expected) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [geo_shape] don't match, values " + prettyPrintCollections(actual, expected) + ) + ); + } + } + /** * Generic matcher that supports common matching logic like null values. */ diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java index 8350ef3ab7a72..96fbef86743b8 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java @@ -95,6 +95,10 @@ public SourceMatcher( new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); put("boolean", new FieldSpecificMatcher.BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put( + "geo_shape", + new FieldSpecificMatcher.GeoShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + ); } }; this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java new file mode 100644 index 0000000000000..60276333f365f --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.test.ESTestCase; + +public class FieldTypeTests extends ESTestCase { + public void testRoundTrip() { + for (var v : FieldType.values()) { + assertEquals(v.toString(), FieldType.tryParse(v.toString()).toString()); + } + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index d984b331e99cb..61d57f19cdfae 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -365,7 +365,7 @@ private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBloc var nullValueEncoded = nullValueFormatted != null ? (Number) unsignedToSortableSignedLong(parseUnsignedLong(nullValueFormatted)) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueFormatted) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueFormatted) { @Override public void convertValue(Object value, List accumulator) { if (value.equals("")) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 62e68bfdb425c..872d174a763b7 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -197,6 +197,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { parser, scriptValues(), geoFormatterFactory, + context.isSourceSynthetic(), meta.get() ); hasScript = script.get() != null; @@ -216,6 +217,7 @@ public static final class GeoShapeWithDocValuesFieldType extends AbstractShapeGe private final GeoFormatterFactory geoFormatterFactory; private final FieldValues scriptValues; + private final boolean isSyntheticSource; public GeoShapeWithDocValuesFieldType( String name, @@ -226,11 +228,13 @@ public GeoShapeWithDocValuesFieldType( GeoShapeParser parser, FieldValues scriptValues, GeoFormatterFactory geoFormatterFactory, + boolean isSyntheticSource, Map meta ) { super(name, indexed, isStored, hasDocValues, parser, orientation, meta); this.scriptValues = scriptValues; this.geoFormatterFactory = geoFormatterFactory; + this.isSyntheticSource = isSyntheticSource; } @Override @@ -306,9 +310,14 @@ protected Function, List> getFormatter(String format) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS - ? new GeoBoundsBlockLoader(name()) - : blockLoaderFromSource(blContext); + if (blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) { + return new GeoBoundsBlockLoader(name()); + } + if (isSyntheticSource) { + return blockLoaderFromFallbackSyntheticSource(blContext); + } + + return blockLoaderFromSource(blContext); } static class GeoBoundsBlockLoader extends AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 3c8127b6c6036..7f66554382d8b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -126,6 +126,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { hasDocValues.get(), orientation.get().value(), parser, + context.isSourceSynthetic(), meta.get() ); return new ShapeFieldMapper(leafName(), ft, builderParams(this, context), parser, this); @@ -142,6 +143,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { ); public static final class ShapeFieldType extends AbstractShapeGeometryFieldType implements ShapeQueryable { + private final boolean isSyntheticSource; public ShapeFieldType( String name, @@ -149,9 +151,11 @@ public ShapeFieldType( boolean hasDocValues, Orientation orientation, Parser parser, + boolean isSyntheticSource, Map meta ) { super(name, indexed, false, hasDocValues, parser, orientation, meta); + this.isSyntheticSource = isSyntheticSource; } @Override @@ -193,9 +197,15 @@ protected Function, List> getFormatter(String format) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS - ? new CartesianBoundsBlockLoader(name()) - : blockLoaderFromSource(blContext); + if (blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) { + return new CartesianBoundsBlockLoader(name()); + } + + if (isSyntheticSource) { + return blockLoaderFromFallbackSyntheticSource(blContext); + } + + return blockLoaderFromSource(blContext); } static class CartesianBoundsBlockLoader extends BoundsBlockLoader { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java new file mode 100644 index 0000000000000..a7419cc0e88e7 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.util.HashMap; + +public class GeoShapeDataSourceHandler implements DataSourceHandler { + @Override + public DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGenerator request) { + return new DataSourceResponse.GeoShapeGenerator(() -> { + while (true) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + if (geometry.type() == ShapeType.ENVELOPE) { + // Not supported in GeoJson, skip it + continue; + } + try { + GeoTestUtils.binaryGeoShapeDocValuesField("f", geometry); + return geometry; + } catch (IllegalArgumentException ignored) { + // Some generated shapes are not suitable for the storage format, ignore them + } + } + }); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType().equals("geo_shape") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var map = new HashMap(); + map.put("store", ESTestCase.randomBoolean()); + map.put("index", ESTestCase.randomBoolean()); + map.put("doc_values", ESTestCase.randomBoolean()); + + if (ESTestCase.randomDouble() <= 0.2) { + // TODO put WKT here ??? + // injected.put("null_value", "yo"); + } + + if (ESTestCase.randomBoolean()) { + map.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return map; + }); + } + + @Override + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + if (request.fieldType().equals("geo_shape") == false) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(new GeoShapeFieldDataGenerator(request.dataSource())); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java new file mode 100644 index 0000000000000..2abb329bef2ea --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.Map; +import java.util.function.Supplier; + +public class GeoShapeFieldDataGenerator implements FieldDataGenerator { + private final Supplier formattedGeoShapes; + + public GeoShapeFieldDataGenerator(DataSource dataSource) { + var geoShapes = dataSource.get(new DataSourceRequest.GeoShapeGenerator()).generator(); + var serializeToGeoJson = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, g -> GeoJson.toMap((Geometry) g))); + + this.formattedGeoShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { + var value = values.get(); + if (value instanceof Geometry g) { + // did not transform + return WellKnownText.toWKT(g); + } + return value; + }).apply(geoShapes::get); + + } + + @Override + public Object generateValue(Map fieldMapping) { + return formattedGeoShapes.get(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..9fb5810f2cdfc --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.common.geo.GeometryNormalizer; +import org.elasticsearch.common.geo.Orientation; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.support.MapXContentParser; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.datageneration.GeoShapeDataSourceHandler; + +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class GeoShapeFieldBlockLoaderTests extends BlockLoaderTestCase { + public GeoShapeFieldBlockLoaderTests(Params params) { + super("geo_shape", List.of(new GeoShapeDataSourceHandler()), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value) { + var nullValue = switch (fieldMapping.get("null_value")) { + case String s -> convert(s, null); + case null -> null; + default -> throw new IllegalStateException("Unexpected null_value format"); + }; + + if (value instanceof List == false) { + return convert(value, nullValue); + } + + // TODO FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS is currently not covered, it needs special logic + // As a result we always load from source (stored or fallback synthetic) and they should work the same. + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + private Object convert(Object value, Object nullValue) { + if (value == null) { + return nullValue; + } + + if (value instanceof String s) { + return toWKB(fromWKT(s)); + } + + if (value instanceof Map m) { + return toWKB(fromGeoJson(m)); + } + + // Malformed values are excluded + return null; + } + + private Geometry fromWKT(String s) { + try { + var geometry = WellKnownText.fromWKT(GeographyValidator.instance(true), false, s); + return normalize(geometry); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private Geometry fromGeoJson(Map map) { + try { + var parser = new MapXContentParser( + xContentRegistry(), + LoggingDeprecationHandler.INSTANCE, + (Map) map, + XContentType.JSON + ); + parser.nextToken(); + + var geometry = GeoJson.fromXContent(GeographyValidator.instance(true), false, true, parser); + return normalize(geometry); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Geometry normalize(Geometry geometry) { + if (GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry)) { + return GeometryNormalizer.apply(Orientation.RIGHT, geometry); + } + + return geometry; + } + + private BytesRef toWKB(Geometry geometry) { + return new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN)); + } + + @Override + protected Collection getPlugins() { + var plugin = new LocalStateSpatialPlugin(); + plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() { + @Override + public List loadExtensions(Class extensionPointType) { + return List.of(); + } + }); + + return Collections.singletonList(plugin); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index a34f0ba2eae3e..9a5dfbfcd9abc 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -569,7 +569,7 @@ protected static Object asWKT(BytesRef value) { @Override protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { // Synthetic source is currently not supported. - return new BlockReaderSupport(false, false, mapper, loaderFieldName); + return new BlockReaderSupport(false, true, mapper, loaderFieldName); } @Override diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 61491b88a8a0b..577e46d91b7b2 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -388,7 +388,7 @@ protected static Object asWKT(BytesRef value) { @Override protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { // Synthetic source is currently not supported. - return new BlockReaderSupport(false, false, mapper, loaderFieldName); + return new BlockReaderSupport(false, true, mapper, loaderFieldName); } @Override diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java index 2713afc149e05..11a0f9ec6f11f 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java @@ -206,6 +206,7 @@ public void testGeoShapeQueryAcrossDateline() throws IOException { null, null, null, + false, Collections.emptyMap() ); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java index 834d04e49014c..3e339ea35b93d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -259,6 +259,7 @@ protected void testCase( null, null, null, + false, Collections.emptyMap() ); testCase( diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java index 8f479c7ed22c3..5515db1b9120e 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java @@ -71,6 +71,7 @@ public void testEmpty() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -100,6 +101,7 @@ public void testUnmappedFieldWithDocs() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -125,6 +127,7 @@ public void testMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); @@ -158,6 +161,7 @@ public void testInvalidMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); @@ -248,6 +252,7 @@ private void readAndAssertExtent(RandomIndexWriter w, TestPointCollection points true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java index 7d4ee29ffee0f..e1d3afe9e3e58 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java @@ -58,6 +58,7 @@ public void testEmpty() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -82,12 +83,21 @@ public void testUnmapped() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); InternalCartesianCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); assertNull(result.centroid()); - fieldType = new ShapeFieldMapper.ShapeFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + fieldType = new ShapeFieldMapper.ShapeFieldType( + "field", + true, + true, + Orientation.RIGHT, + null, + false, + Collections.emptyMap() + ); result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); @@ -112,6 +122,7 @@ public void testUnmappedWithMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); InternalCartesianCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -185,6 +196,7 @@ private void assertCentroid(RandomIndexWriter w, CartesianPoint expectedCentroid true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); CartesianCentroidAggregationBuilder aggBuilder = new CartesianCentroidAggregationBuilder("my_agg").field("field"); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java index 50eb9fda9e818..027408c273bd3 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java @@ -62,6 +62,7 @@ public void testEmpty() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -96,6 +97,7 @@ public void testUnmappedFieldWithDocs() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -126,6 +128,7 @@ public void testMissing() throws Exception { null, null, null, + false, Collections.emptyMap() ); @@ -165,6 +168,7 @@ public void testInvalidMissing() throws Exception { null, null, null, + false, Collections.emptyMap() ); @@ -230,6 +234,7 @@ public void testRandomShapes() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java index 86ce455d372e7..c27bbc61f01e2 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java @@ -67,6 +67,7 @@ public void testEmpty() throws Exception { null, null, null, + false, Map.of() ); try (IndexReader reader = w.getReader()) { @@ -94,6 +95,7 @@ public void testUnmapped() throws Exception { null, null, null, + false, Map.of() ); InternalGeoCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -108,6 +110,7 @@ public void testUnmapped() throws Exception { null, null, null, + false, Map.of() ); result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -138,6 +141,7 @@ public void testUnmappedWithMissing() throws Exception { null, null, null, + false, Map.of() ); InternalGeoCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -214,6 +218,7 @@ private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) thro null, null, null, + false, Map.of() ); GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg").field("field"); From cd900e6873a652ce7564150a90811807b2168006 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 14 Mar 2025 14:09:03 -0700 Subject: [PATCH 2/6] integrate into challenge tests --- .../mapper/AbstractGeometryFieldMapper.java | 16 ++- .../datageneration/MappingGenerator.java | 2 +- .../datasource/DataSourceHandler.java | 4 + .../datasource/DataSourceRequest.java | 6 + .../datasource/DataSourceResponse.java | 5 +- .../DefaultObjectGenerationHandler.java | 2 +- .../datageneration/fields/leaf/Wrappers.java | 8 +- .../datageneration/matchers/Matcher.java | 21 +++- .../matchers/source/FieldSpecificMatcher.java | 6 +- .../matchers/source/SourceMatcher.java | 15 ++- .../matchers/source/SourceTransforms.java | 38 +++++-- x-pack/plugin/logsdb/build.gradle | 1 + .../xpack/logsdb/qa/DataGenerationHelper.java | 46 +++++++- ...ndexedIntoStandardModeChallengeRestIT.java | 4 +- ...ndexedIntoStoredSourceChallengeRestIT.java | 4 +- ...bVersusReindexedLogsDbChallengeRestIT.java | 4 +- ...ardVersusLogsIndexModeChallengeRestIT.java | 10 +- ...ardReindexedIntoLogsDbChallengeRestIT.java | 4 +- ...bVersusReindexedLogsDbChallengeRestIT.java | 4 +- x-pack/plugin/spatial/build.gradle | 1 + .../GeoShapeDataSourceHandler.java | 38 +++---- .../GeoShapeFieldDataGenerator.java | 25 +++-- .../ShapeDataSourceHandler.java | 70 ++++++++++++ .../ShapeFieldDataGenerator.java | 56 ++++++++++ .../mapper/GeoShapeFieldBlockLoaderTests.java | 16 +-- ...GeoShapeWithDocValuesFieldMapperTests.java | 2 +- .../mapper/ShapeFieldBlockLoaderTests.java | 104 ++++++++++++++++++ .../index/mapper/ShapeFieldMapperTests.java | 2 +- 28 files changed, 420 insertions(+), 94 deletions(-) create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 8fc91c435cb97..0a51564a4cff5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -217,7 +217,13 @@ private GeometriesFallbackSyntheticSourceReader(Parser geometryParser, Functi public void convertValue(Object value, List accumulator) { final List values = new ArrayList<>(); - geometryParser.fetchFromSource(value, values::add); + geometryParser.fetchFromSource(value, v -> { + if (v != null) { + values.add(v); + } else if (nullValue != null) { + values.add(nullValue); + } + }); var formatted = formatter.apply(values); for (var formattedValue : formatted) { @@ -235,7 +241,13 @@ public void convertValue(Object value, List accumulator) { public void parse(XContentParser parser, List accumulator) throws IOException { final List values = new ArrayList<>(); - geometryParser.parseFromSource(parser, values::add); + geometryParser.parseFromSource(parser, v -> { + if (v != null) { + values.add(v); + } else if (nullValue != null) { + values.add(nullValue); + } + }); var formatted = formatter.apply(values); for (var formattedValue : formatted) { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java index da26e9957198d..d5ec1c7f9f36a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java @@ -112,7 +112,7 @@ private void generateMapping( ) .mappingGenerator(); - mappingParameters.put("type", leaf.type().toString()); + mappingParameters.put("type", leaf.type()); mappingParameters.putAll(mappingParametersGenerator.get()); // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 829d2ae0d24a9..e4a3c9579595a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -62,6 +62,10 @@ default DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGe return null; } + default DataSourceResponse.ShapeGenerator handle(DataSourceRequest.ShapeGenerator request) { + return null; + } + default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 47787fb6abc0c..10d12acf2a5d5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -100,6 +100,12 @@ public DataSourceResponse.GeoShapeGenerator accept(DataSourceHandler handler) { } } + record ShapeGenerator() implements DataSourceRequest { + public DataSourceResponse.ShapeGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record NullWrapper() implements DataSourceRequest { public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) { return handler.handle(this); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index bbe98ccc8e5ca..780b9a8fe6cc5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -10,7 +10,6 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.logsdb.datageneration.FieldType; import java.time.Instant; import java.util.Map; @@ -45,6 +44,8 @@ record InstantGenerator(Supplier generator) implements DataSourceRespon record GeoShapeGenerator(Supplier generator) implements DataSourceResponse {} + record ShapeGenerator(Supplier generator) implements DataSourceResponse {} + record NullWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} @@ -68,7 +69,7 @@ interface ChildFieldGenerator extends DataSourceResponse { } record FieldTypeGenerator(Supplier generator) implements DataSourceResponse { - public record FieldTypeInfo(FieldType fieldType) {} + public record FieldTypeInfo(String fieldType) {} } record ObjectArrayGenerator(Supplier> lengthGenerator) implements DataSourceResponse {} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java index 56ec676e53d55..29ffd8d965712 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java @@ -62,7 +62,7 @@ public String generateFieldName() { public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values())) + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values()).toString()) ); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java index 74be106620d46..348e8fdad098a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java @@ -19,7 +19,7 @@ public class Wrappers { * Applies default wrappers for raw values - adds nulls and wraps values in arrays. * @return */ - static Supplier defaults(Supplier rawValues, DataSource dataSource) { + public static Supplier defaults(Supplier rawValues, DataSource dataSource) { var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); @@ -30,7 +30,11 @@ static Supplier defaults(Supplier rawValues, DataSource dataSour * Applies default wrappers for raw values and also adds malformed values. * @return */ - static Supplier defaultsWithMalformed(Supplier rawValues, Supplier malformedValues, DataSource dataSource) { + public static Supplier defaultsWithMalformed( + Supplier rawValues, + Supplier malformedValues, + DataSource dataSource + ) { var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var malformed = dataSource.get(new DataSourceRequest.MalformedWrapper(malformedValues)); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java index dd87e23351c0d..7fa82dea1a638 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java @@ -30,7 +30,11 @@ public static MappingsStep>> matchSource() { } public interface MappingsStep { - SettingsStep mappings(XContentBuilder actualMappings, XContentBuilder expectedMappings); + SettingsStep mappings( + Map> mappingLookup, + XContentBuilder actualMappings, + XContentBuilder expectedMappings + ); } public interface SettingsStep { @@ -104,6 +108,7 @@ private static class SourceMatcherBuilder SettingsStep>>, CompareStep>>, ExpectedStep>> { + private Map> mappingLookup; private XContentBuilder expectedMappings; private XContentBuilder actualMappings; private Settings.Builder expectedSettings; @@ -121,9 +126,11 @@ public ExpectedStep>> settings(Settings.Builder actualS private SourceMatcherBuilder() {} public SettingsStep>> mappings( + final Map> mappingLookup, final XContentBuilder actualMappings, final XContentBuilder expectedMappings ) { + this.mappingLookup = mappingLookup; this.actualMappings = actualMappings; this.expectedMappings = expectedMappings; @@ -132,8 +139,16 @@ public SettingsStep>> mappings( @Override public MatchResult isEqualTo(List> actual) { - return new SourceMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort) - .match(); + return new SourceMatcher( + mappingLookup, + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + actual, + expected, + ignoringSort + ).match(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index 1448019d64a8a..5612b02a3b562 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -473,13 +473,13 @@ Object convert(Object value, Object nullValue, DateTimeFormatter dateTimeFormatt } } - class GeoShapeMatcher implements FieldSpecificMatcher { + class ShapeMatcher implements FieldSpecificMatcher { private final XContentBuilder actualMappings; private final Settings.Builder actualSettings; private final XContentBuilder expectedMappings; private final Settings.Builder expectedSettings; - GeoShapeMatcher( + ShapeMatcher( XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, @@ -498,7 +498,7 @@ public MatchResult match( Map actualMapping, Map expectedMapping ) { - // Since fallback synthetic source is used, should match exactly + // Since fallback synthetic source is used, should always match exactly. return actual.equals(expected) ? MatchResult.match() : MatchResult.noMatch( diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java index 96fbef86743b8..f5774288e0d1f 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java @@ -25,6 +25,8 @@ import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintCollections; public class SourceMatcher extends GenericEqualsMatcher>> { + private final Map> mappingLookup; + private final Map actualNormalizedMapping; private final Map expectedNormalizedMapping; @@ -32,6 +34,7 @@ public class SourceMatcher extends GenericEqualsMatcher private final DynamicFieldMatcher dynamicFieldMatcher; public SourceMatcher( + final Map> mappingLookup, final XContentBuilder actualMappings, final Settings.Builder actualSettings, final XContentBuilder expectedMappings, @@ -42,6 +45,8 @@ public SourceMatcher( ) { super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort); + this.mappingLookup = mappingLookup; + var actualMappingAsMap = XContentHelper.convertToMap(BytesReference.bytes(actualMappings), false, actualMappings.contentType()) .v2(); this.actualNormalizedMapping = MappingTransforms.normalizeMapping(actualMappingAsMap); @@ -95,10 +100,8 @@ public SourceMatcher( new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); put("boolean", new FieldSpecificMatcher.BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); - put( - "geo_shape", - new FieldSpecificMatcher.GeoShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) - ); + put("geo_shape", new FieldSpecificMatcher.ShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("shape", new FieldSpecificMatcher.ShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); } }; this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); @@ -118,8 +121,8 @@ public MatchResult match() { ); } - var sortedAndFlattenedActual = actual.stream().map(SourceTransforms::normalize).toList(); - var sortedAndFlattenedExpected = expected.stream().map(SourceTransforms::normalize).toList(); + var sortedAndFlattenedActual = actual.stream().map(s -> SourceTransforms.normalize(s, mappingLookup)).toList(); + var sortedAndFlattenedExpected = expected.stream().map(s -> SourceTransforms.normalize(s, mappingLookup)).toList(); for (int i = 0; i < sortedAndFlattenedActual.size(); i++) { var actual = sortedAndFlattenedActual.get(i); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java index c86fe2f90b0d1..e3eb80799c416 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java @@ -31,10 +31,10 @@ class SourceTransforms { * * @return flattened map */ - public static Map> normalize(Map map) { + public static Map> normalize(Map documentMap, Map> mappingLookup) { var flattened = new TreeMap>(); - descend(null, map, flattened); + descend(null, documentMap, flattened, mappingLookup); return flattened; } @@ -55,28 +55,44 @@ public static List normalizeValues(List values, Function tran // Synthetic source modifications: // * null values are not present // * duplicates are removed - return new ArrayList<>( - values.stream().filter(v -> v != null && Objects.equals(v, "null") == false).map(transform).collect(Collectors.toSet()) - ); + return values.stream() + .filter(v -> v != null && Objects.equals(v, "null") == false) + .map(transform) + .distinct() + .collect(Collectors.toList()); } - private static void descend(String pathFromRoot, Map currentLevel, Map> flattened) { + private static void descend( + String pathFromRoot, + Map currentLevel, + Map> flattened, + Map> mappingLookup + ) { for (var entry : currentLevel.entrySet()) { var pathToCurrentField = pathFromRoot == null ? entry.getKey() : pathFromRoot + "." + entry.getKey(); if (entry.getValue() instanceof List list) { for (var fieldValue : list) { - handleField(pathToCurrentField, fieldValue, flattened); + handleField(pathToCurrentField, fieldValue, flattened, mappingLookup); } } else { - handleField(pathToCurrentField, entry.getValue(), flattened); + handleField(pathToCurrentField, entry.getValue(), flattened, mappingLookup); } } } @SuppressWarnings("unchecked") - private static void handleField(String pathToCurrentField, Object currentField, Map> flattened) { - if (currentField instanceof Map map) { - descend(pathToCurrentField, (Map) map, flattened); + private static void handleField( + String pathToCurrentField, + Object currentField, + Map> flattened, + Map> mappingLookup + ) { + var mapping = mappingLookup.get(pathToCurrentField); + // Values of some fields are complex objects so we need to double-check that this is actually and object or a nested field + // we can descend into. + if (currentField instanceof Map map + && (mapping == null || mapping.get("type").equals("object") || mapping.get("type").equals("nested"))) { + descend(pathToCurrentField, (Map) map, flattened, mappingLookup); } else { flattened.computeIfAbsent(pathToCurrentField, k -> new ArrayList<>()).add(currentField); } diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index bef07258a8e33..9adab25e2c5e4 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -32,6 +32,7 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation project(':modules:data-streams') testImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(xpackModule('spatial')))) internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index fb890f3ac7ae7..1cba221af14d4 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -16,14 +16,22 @@ import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; import org.elasticsearch.logsdb.datageneration.TemplateGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.spatial.datageneration.GeoShapeDataSourceHandler; +import org.elasticsearch.xpack.spatial.datageneration.ShapeDataSourceHandler; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; public class DataGenerationHelper { private final boolean keepArraySource; @@ -67,7 +75,35 @@ public DataGenerationHelper(Consumer builder (ignored) -> ESTestCase.randomLongBetween(1000, 2000) ) ) - ); + ) + // "shape" and "geo_shape" are very similar so we'll use only one of them + .withDataSourceHandlers(List.of(new GeoShapeDataSourceHandler(), new ShapeDataSourceHandler())) + .withDataSourceHandlers(List.of(new DataSourceHandler() { + @Override + public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { + return new DataSourceResponse.FieldTypeGenerator(new Supplier<>() { + // geo_shape and shape tends to produce really big values so let's limit how many we generate + private int shapesGenerated = 0; + + @Override + public DataSourceResponse.FieldTypeGenerator.FieldTypeInfo get() { + var options = Arrays.stream(FieldType.values()).map(FieldType::toString).collect(Collectors.toSet()); + // Custom types coming from specific functionality modules + if (shapesGenerated < 5) { + options.add("geo_shape"); + options.add("shape"); + } + + var randomChoice = ESTestCase.randomFrom(options); + if (randomChoice.equals("geo_shape") || randomChoice.equals("shape")) { + shapesGenerated++; + } + + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(options)); + } + }); + } + })); // Customize builder if necessary builderConfigurator.accept(specificationBuilder); @@ -80,11 +116,15 @@ public DataGenerationHelper(Consumer builder this.mapping = new MappingGenerator(specification).generate(template); } - void logsDbMapping(XContentBuilder builder) throws IOException { + Mapping mapping() { + return this.mapping; + } + + void writeLogsDbMapping(XContentBuilder builder) throws IOException { builder.map(mapping.raw()); } - void standardMapping(XContentBuilder builder) throws IOException { + void writeStandardMapping(XContentBuilder builder) throws IOException { builder.map(mapping.raw()); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java index d9abdc2cde446..490f653c3ec16 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java index 776a6faf7fa07..ceaeb7cf362d6 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java index 8b00c647b5dd0..1a90dfbbd8ac6 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 2a8c85efc863b..29fe49cffee16 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -65,12 +65,12 @@ protected StandardVersusLogsIndexModeChallengeRestIT(DataGenerationHelper dataGe @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override @@ -130,7 +130,7 @@ public void testMatchAllQuery() throws IOException { .size(numberOfDocuments); final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getQueryHits(queryBaseline(searchSourceBuilder))) .ignoringSort(true) @@ -148,7 +148,7 @@ public void testTermsQuery() throws IOException { .size(numberOfDocuments); final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getQueryHits(queryBaseline(searchSourceBuilder))) .ignoringSort(true) @@ -218,7 +218,7 @@ public void testEsqlSource() throws IOException { final String query = "FROM $index METADATA _source, _id | KEEP _source, _id | LIMIT " + numberOfDocuments; final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getEsqlSourceResults(esqlBaseline(query))) .ignoringSort(true) diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java index 5adf44f10be45..d84f12e73e5b2 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java index a0672daafb243..c67707b6c3f2f 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java @@ -39,11 +39,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index e009f5e6be0ff..3b3c2cc71d843 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -7,6 +7,7 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-test-artifact' esplugin { name = 'spatial' diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java index a7419cc0e88e7..183bb4b22d551 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.spatial.datageneration; import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; @@ -20,21 +21,7 @@ public class GeoShapeDataSourceHandler implements DataSourceHandler { @Override public DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGenerator request) { - return new DataSourceResponse.GeoShapeGenerator(() -> { - while (true) { - var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); - if (geometry.type() == ShapeType.ENVELOPE) { - // Not supported in GeoJson, skip it - continue; - } - try { - GeoTestUtils.binaryGeoShapeDocValuesField("f", geometry); - return geometry; - } catch (IllegalArgumentException ignored) { - // Some generated shapes are not suitable for the storage format, ignore them - } - } - }); + return new DataSourceResponse.GeoShapeGenerator(this::generateValidGeoShape); } @Override @@ -49,11 +36,6 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques map.put("index", ESTestCase.randomBoolean()); map.put("doc_values", ESTestCase.randomBoolean()); - if (ESTestCase.randomDouble() <= 0.2) { - // TODO put WKT here ??? - // injected.put("null_value", "yo"); - } - if (ESTestCase.randomBoolean()) { map.put("ignore_malformed", ESTestCase.randomBoolean()); } @@ -70,4 +52,20 @@ public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataG return new DataSourceResponse.FieldDataGenerator(new GeoShapeFieldDataGenerator(request.dataSource())); } + + private Geometry generateValidGeoShape() { + while (true) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + if (geometry.type() == ShapeType.ENVELOPE) { + // Not supported in GeoJson, skip it + continue; + } + try { + GeoTestUtils.binaryGeoShapeDocValuesField("f", geometry); + return geometry; + } catch (IllegalArgumentException ignored) { + // Some generated shapes are not suitable for the storage format, ignore them + } + } + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java index 2abb329bef2ea..17ecbde73ad71 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java @@ -5,15 +5,6 @@ * 2.0. */ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - package org.elasticsearch.xpack.spatial.datageneration; import org.elasticsearch.common.geo.GeoJson; @@ -22,18 +13,20 @@ import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.fields.leaf.Wrappers; import java.util.Map; import java.util.function.Supplier; public class GeoShapeFieldDataGenerator implements FieldDataGenerator { private final Supplier formattedGeoShapes; + private final Supplier formattedGeoShapesWithMalformed; public GeoShapeFieldDataGenerator(DataSource dataSource) { var geoShapes = dataSource.get(new DataSourceRequest.GeoShapeGenerator()).generator(); var serializeToGeoJson = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, g -> GeoJson.toMap((Geometry) g))); - this.formattedGeoShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { + var formattedGeoShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { var value = values.get(); if (value instanceof Geometry g) { // did not transform @@ -41,11 +34,23 @@ public GeoShapeFieldDataGenerator(DataSource dataSource) { } return value; }).apply(geoShapes::get); + this.formattedGeoShapes = Wrappers.defaults(formattedGeoShapes, dataSource); + var longs = dataSource.get(new DataSourceRequest.LongGenerator()).generator(); + this.formattedGeoShapesWithMalformed = Wrappers.defaultsWithMalformed(formattedGeoShapes, longs::get, dataSource); } @Override public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // ??? + return null; + } + + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return formattedGeoShapesWithMalformed.get(); + } + return formattedGeoShapes.get(); } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java new file mode 100644 index 0000000000000..eb6bb1b9b34e0 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.util.HashMap; + +public class ShapeDataSourceHandler implements DataSourceHandler { + @Override + public DataSourceResponse.ShapeGenerator handle(DataSourceRequest.ShapeGenerator request) { + return new DataSourceResponse.ShapeGenerator(this::generateValidShape); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType().equals("shape") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var map = new HashMap(); + map.put("index", ESTestCase.randomBoolean()); + map.put("doc_values", ESTestCase.randomBoolean()); + + if (ESTestCase.randomBoolean()) { + map.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return map; + }); + } + + @Override + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + if (request.fieldType().equals("shape") == false) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(new ShapeFieldDataGenerator(request.dataSource())); + } + + private Geometry generateValidShape() { + while (true) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + if (geometry.type() == ShapeType.ENVELOPE) { + // Not supported in GeoJson, skip it + continue; + } + try { + GeoTestUtils.binaryCartesianShapeDocValuesField("f", geometry); + return geometry; + } catch (IllegalArgumentException ignored) { + // Some generated shapes are not suitable for the storage format, ignore them + } + } + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java new file mode 100644 index 0000000000000..d9e4c6d579321 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.fields.leaf.Wrappers; + +import java.util.Map; +import java.util.function.Supplier; + +public class ShapeFieldDataGenerator implements FieldDataGenerator { + private final Supplier formattedShapes; + private final Supplier formattedShapesWithMalformed; + + public ShapeFieldDataGenerator(DataSource dataSource) { + var shapes = dataSource.get(new DataSourceRequest.ShapeGenerator()).generator(); + var serializeToGeoJson = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, g -> GeoJson.toMap((Geometry) g))); + + var formattedShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { + var value = values.get(); + if (value instanceof Geometry g) { + // did not transform + return WellKnownText.toWKT(g); + } + return value; + }).apply(shapes::get); + this.formattedShapes = Wrappers.defaults(formattedShapes, dataSource); + + var longs = dataSource.get(new DataSourceRequest.LongGenerator()).generator(); + this.formattedShapesWithMalformed = Wrappers.defaultsWithMalformed(formattedShapes, longs::get, dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // ??? + return null; + } + + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return formattedShapesWithMalformed.get(); + } + + return formattedShapes.get(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java index 9fb5810f2cdfc..2c9a70a66514f 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java @@ -39,27 +39,17 @@ public GeoShapeFieldBlockLoaderTests(Params params) { @Override @SuppressWarnings("unchecked") protected Object expected(Map fieldMapping, Object value) { - var nullValue = switch (fieldMapping.get("null_value")) { - case String s -> convert(s, null); - case null -> null; - default -> throw new IllegalStateException("Unexpected null_value format"); - }; - if (value instanceof List == false) { - return convert(value, nullValue); + return convert(value); } // TODO FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS is currently not covered, it needs special logic // As a result we always load from source (stored or fallback synthetic) and they should work the same. - var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); + var resultList = ((List) value).stream().map(this::convert).filter(Objects::nonNull).toList(); return maybeFoldList(resultList); } - private Object convert(Object value, Object nullValue) { - if (value == null) { - return nullValue; - } - + private Object convert(Object value) { if (value instanceof String s) { return toWKB(fromWKT(s)); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 9a5dfbfcd9abc..a34f0ba2eae3e 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -569,7 +569,7 @@ protected static Object asWKT(BytesRef value) { @Override protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { // Synthetic source is currently not supported. - return new BlockReaderSupport(false, true, mapper, loaderFieldName); + return new BlockReaderSupport(false, false, mapper, loaderFieldName); } @Override diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..3acd64c501a4a --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.StandardValidator; +import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.support.MapXContentParser; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.datageneration.ShapeDataSourceHandler; + +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ShapeFieldBlockLoaderTests extends BlockLoaderTestCase { + public ShapeFieldBlockLoaderTests(Params params) { + super("shape", List.of(new ShapeDataSourceHandler()), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value) { + if (value instanceof List == false) { + return convert(value); + } + + // TODO FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS is currently not covered, it needs special logic + // As a result we always load from source (stored or fallback synthetic) and they should work the same. + var resultList = ((List) value).stream().map(this::convert).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + private Object convert(Object value) { + if (value instanceof String s) { + return toWKB(fromWKT(s)); + } + + if (value instanceof Map m) { + return toWKB(fromGeoJson(m)); + } + + // Malformed values are excluded + return null; + } + + private Geometry fromWKT(String s) { + try { + return WellKnownText.fromWKT(StandardValidator.instance(true), false, s); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private Geometry fromGeoJson(Map map) { + try { + var parser = new MapXContentParser( + xContentRegistry(), + LoggingDeprecationHandler.INSTANCE, + (Map) map, + XContentType.JSON + ); + parser.nextToken(); + + return GeoJson.fromXContent(StandardValidator.instance(true), false, true, parser); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private BytesRef toWKB(Geometry geometry) { + return new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN)); + } + + @Override + protected Collection getPlugins() { + var plugin = new LocalStateSpatialPlugin(); + plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() { + @Override + public List loadExtensions(Class extensionPointType) { + return List.of(); + } + }); + + return Collections.singletonList(plugin); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 577e46d91b7b2..61491b88a8a0b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -388,7 +388,7 @@ protected static Object asWKT(BytesRef value) { @Override protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { // Synthetic source is currently not supported. - return new BlockReaderSupport(false, true, mapper, loaderFieldName); + return new BlockReaderSupport(false, false, mapper, loaderFieldName); } @Override From ecaaf7f7a87ed3c9c85cc2686e822907fe0d609f Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 14 Mar 2025 14:43:38 -0700 Subject: [PATCH 3/6] iter --- .../index/mapper/AbstractGeometryFieldMapper.java | 10 +++------- .../datasource/DefaultFieldDataGeneratorHandler.java | 1 + .../xpack/logsdb/qa/DataGenerationHelper.java | 5 +++-- .../datageneration/GeoShapeFieldDataGenerator.java | 3 ++- .../datageneration/ShapeFieldDataGenerator.java | 3 ++- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 0a51564a4cff5..894c053e3c0c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -194,9 +194,7 @@ protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { protected abstract Object nullValueAsSource(T nullValue); protected BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { - Function, List> formatter = getFormatter(GeometryFormatterFactory.WKB); - - return new FallbackSyntheticSourceBlockLoader(new GeometriesFallbackSyntheticSourceReader(geometryParser, formatter), name()) { + return new FallbackSyntheticSourceBlockLoader(new GeometriesFallbackSyntheticSourceReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { return factory.bytesRefs(expectedCount); @@ -205,12 +203,10 @@ public Builder builder(BlockFactory factory, int expectedCount) { } private class GeometriesFallbackSyntheticSourceReader implements FallbackSyntheticSourceBlockLoader.Reader { - private final Parser geometryParser; private final Function, List> formatter; - private GeometriesFallbackSyntheticSourceReader(Parser geometryParser, Function, List> formatter) { - this.geometryParser = geometryParser; - this.formatter = formatter; + private GeometriesFallbackSyntheticSourceReader() { + this.formatter = getFormatter(GeometryFormatterFactory.WKB); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java index 62578ec08cbcf..7287c0edd6b69 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java @@ -15,6 +15,7 @@ public class DefaultFieldDataGeneratorHandler implements DataSourceHandler { public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { var fieldType = FieldType.tryParse(request.fieldType()); if (fieldType == null) { + // This is a custom field type return null; } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index 1cba221af14d4..f66188f1ecb1f 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -76,7 +76,6 @@ public DataGenerationHelper(Consumer builder ) ) ) - // "shape" and "geo_shape" are very similar so we'll use only one of them .withDataSourceHandlers(List.of(new GeoShapeDataSourceHandler(), new ShapeDataSourceHandler())) .withDataSourceHandlers(List.of(new DataSourceHandler() { @Override @@ -87,8 +86,10 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG @Override public DataSourceResponse.FieldTypeGenerator.FieldTypeInfo get() { + // Base set of field types var options = Arrays.stream(FieldType.values()).map(FieldType::toString).collect(Collectors.toSet()); // Custom types coming from specific functionality modules + if (shapesGenerated < 5) { options.add("geo_shape"); options.add("shape"); @@ -96,7 +97,7 @@ public DataSourceResponse.FieldTypeGenerator.FieldTypeInfo get() { var randomChoice = ESTestCase.randomFrom(options); if (randomChoice.equals("geo_shape") || randomChoice.equals("shape")) { - shapesGenerated++; + shapesGenerated += 1; } return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(options)); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java index 17ecbde73ad71..a8650b50def45 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java @@ -43,7 +43,8 @@ public GeoShapeFieldDataGenerator(DataSource dataSource) { @Override public Object generateValue(Map fieldMapping) { if (fieldMapping == null) { - // ??? + // dynamically mapped and dynamic mapping does not play well with this type (it gets mapped as an object) + // return null to skip indexing this field return null; } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java index d9e4c6d579321..78f1e8ee1745e 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java @@ -43,7 +43,8 @@ public ShapeFieldDataGenerator(DataSource dataSource) { @Override public Object generateValue(Map fieldMapping) { if (fieldMapping == null) { - // ??? + // dynamically mapped and dynamic mapping does not play well with this type (it gets mapped as an object) + // return null to skip indexing this field return null; } From ee0d56d0c9f730a10013e9adae0d54452454c95d Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 14 Mar 2025 14:46:14 -0700 Subject: [PATCH 4/6] Update docs/changelog/124927.yaml --- docs/changelog/124927.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/124927.yaml diff --git a/docs/changelog/124927.yaml b/docs/changelog/124927.yaml new file mode 100644 index 0000000000000..3da4974a02f82 --- /dev/null +++ b/docs/changelog/124927.yaml @@ -0,0 +1,5 @@ +pr: 124927 +summary: Use `FallbackSyntheticSourceBlockLoader` for shape and `geo_shape` +area: Mapping +type: enhancement +issues: [] From b862d0368f2d4c3d5ed2a870258536207d5a2481 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 14 Mar 2025 14:46:59 -0700 Subject: [PATCH 5/6] Update 124927.yaml --- docs/changelog/124927.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/124927.yaml b/docs/changelog/124927.yaml index 3da4974a02f82..7b3c5b4663ff0 100644 --- a/docs/changelog/124927.yaml +++ b/docs/changelog/124927.yaml @@ -1,5 +1,5 @@ pr: 124927 -summary: Use `FallbackSyntheticSourceBlockLoader` for shape and `geo_shape` +summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` area: Mapping type: enhancement issues: [] From d43f71fcf797c18de0de12687b9f2f2c76ebd02c Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 14 Mar 2025 15:07:57 -0700 Subject: [PATCH 6/6] fix --- .../blockloader/BooleanFieldBlockLoaderTests.java | 2 +- .../mapper/blockloader/DateFieldBlockLoaderTests.java | 2 +- .../blockloader/KeywordFieldBlockLoaderTests.java | 2 +- .../matchers/source/MappingTransforms.java | 2 +- .../datageneration/DataGenerationSnapshotTests.java | 8 ++++---- .../logsdb/datageneration/DataGenerationTests.java | 4 ++-- .../logsdb/datageneration/SourceMatcherTests.java | 10 ++++++---- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java index cb01d8318e471..c1197bcd30d2f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java @@ -18,7 +18,7 @@ public class BooleanFieldBlockLoaderTests extends BlockLoaderTestCase { public BooleanFieldBlockLoaderTests(Params params) { - super(FieldType.BOOLEAN, params); + super(FieldType.BOOLEAN.toString(), params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java index c2aa8282b7c58..738cc68cdab2a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java @@ -24,7 +24,7 @@ public class DateFieldBlockLoaderTests extends BlockLoaderTestCase { public DateFieldBlockLoaderTests(Params params) { - super(FieldType.DATE, params); + super(FieldType.DATE.toString(), params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index 0b4eaf9c0b5f1..f55e516ca9fdb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -22,7 +22,7 @@ public class KeywordFieldBlockLoaderTests extends BlockLoaderTestCase { public KeywordFieldBlockLoaderTests(Params params) { - super(FieldType.KEYWORD, params); + super(FieldType.KEYWORD.toString(), params); } @SuppressWarnings("unchecked") diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java index 312be273dcd3e..2d961f1f1408e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java @@ -26,7 +26,7 @@ record FieldMapping(Map mappingParameters, List { if (fieldType == FieldType.KEYWORD) { fieldType = FieldType.LONG; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD.toString()); } fieldType = FieldType.KEYWORD; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG.toString()); }); } @@ -235,11 +235,11 @@ public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.Dynam @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { - if (request.fieldType() == FieldType.KEYWORD) { + if (request.fieldType().equals(FieldType.KEYWORD.toString())) { return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("store", "true")); } - if (request.fieldType() == FieldType.LONG) { + if (request.fieldType().equals(FieldType.LONG.toString())) { return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("index", "false")); } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java index f5ba8bd02fa88..ee282425b7855 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java @@ -91,7 +91,7 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo( - FieldType.values()[generatedFields++ % FieldType.values().length] + FieldType.values()[generatedFields++ % FieldType.values().length].toString() ) ); @@ -166,7 +166,7 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr @Override public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG) + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG.toString()) ); } }; diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java index aba5cbc9878e1..342394fe14b86 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java @@ -27,6 +27,7 @@ public void testDynamicMatch() throws IOException { ); var sut = new SourceMatcher( + Map.of(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), Settings.builder(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), @@ -49,6 +50,7 @@ public void testDynamicMismatch() throws IOException { ); var sut = new SourceMatcher( + Map.of(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), Settings.builder(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), @@ -77,7 +79,7 @@ public void testMappedMatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), values, values, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), values, values, false); assertTrue(sut.match().isMatch()); } @@ -102,7 +104,7 @@ public void testMappedMismatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertFalse(sut.match().isMatch()); } @@ -119,7 +121,7 @@ public void testCountedKeywordMatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertTrue(sut.match().isMatch()); } @@ -136,7 +138,7 @@ public void testCountedKeywordMismatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertFalse(sut.match().isMatch()); } }