diff --git a/.github/workflows/early-access.yml b/.github/workflows/early-access.yml index 5103bfb..8295df1 100644 --- a/.github/workflows/early-access.yml +++ b/.github/workflows/early-access.yml @@ -91,4 +91,4 @@ jobs: uses: docker/build-push-action@v3 with: push: ${{ github.event_name != 'pull_request' }} - tags: jruaux/trino-redisearch:latest + tags: jruaux/trino-redisearch:early-access diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0441e1c..e2c6c74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,10 @@ on: version: description: "Release version" required: true + ref: + description: "Branch, tag or SHA to checkout" + required: false + default: 'master' jobs: release: name: Release @@ -14,6 +18,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - name: Set up Java @@ -61,7 +66,7 @@ jobs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "GitHub Action" git commit -a -m "Releasing version $VERSION" - git push origin master + git push origin ${{ github.event.inputs.ref }} - name: JReleaser output if: always() @@ -71,3 +76,21 @@ jobs: path: | out/jreleaser/trace.log out/jreleaser/output.properties + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: ${{ github.event_name != 'pull_request' && github.event.inputs.ref == 'master' }} + tags: jruaux/trino-redisearch:latest,jruaux/trino-redisearch:${{ github.event.inputs.version }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 56142d4..f46019f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG TRINO_VERSION=396 +ARG TRINO_VERSION=380 FROM docker.io/library/maven:3.8.6-openjdk-18 AS builder WORKDIR /root/trino-redisearch diff --git a/README.adoc b/README.adoc index e30457e..7fc791e 100644 --- a/README.adoc +++ b/README.adoc @@ -3,167 +3,295 @@ :project-owner: redis-field-engineering :project-name: redis-sql :project-group: com.redis -:project-version: 0.2.5 +:project-version: 0.2.5-380 +:project-url: https://github.com/{project-owner}/{project-name} :artifact-id: trino-redisearch -:codecov-token: CQFF495IPH +:trino-dir: /usr/lib/trino +:trino-datadir: /var/trino +:trino-version: 380 -image:https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml/badge.svg["Build Status", link="https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml"] +image:{project-url}/actions/workflows/early-access.yml/badge.svg["Build Status", link="{project-url}/actions/workflows/early-access.yml"] image:https://codecov.io/gh/{project-owner}/{project-name}/branch/master/graph/badge.svg?token={codecov-token}["Coverage", link="https://codecov.io/gh/{project-owner}/{project-name}"] -Redis SQL is a SQL interface to https://oss.redislabs.com/redisearch/[RediSearch] implemented as a https://trino.io[Trino] connector. -It allows the use of RediSearch indexes as tables in Trino. +Redis SQL is a https://trino.io[Trino] connector which allows access to RediSearch data from Trino. +This document describes how to set up the RediSearch Connector to run SQL queries against RediSearch. -== Requirements +image::https://asciinema.org/a/526185.svg[asciicast,link=https://asciinema.org/a/526185] -To connect to RediSearch you need: - -. RediSearch 2.0 or higher -. Network access from the Trino coordinator and workers to RediSearch. +NOTE: RediSearch 2.0 or later is required. == Configuration - -To configure the RediSearch connector, create a catalog properties file `etc/catalog/redisearch.properties` with the following contents (replace properties as appropriate): +To configure the RediSearch connector, create a catalog properties file `etc/catalog/redisearch.properties` with the following contents, replacing the properties as appropriate: [source,properties] ---- connector.name=redisearch -redisearch.uri=redis://user:pass@host:6379 +redisearch.uri=redis://localhost:6379 +redisearch.default-schema-name=default ---- -=== Configuration Properties +[[properties]] +=== Configuration properties + +[cols="1,1,1"] +|=== +|Property name |Description |Default + +|`redisearch.uri` +|A Redis connection string. https://github.com/lettuce-io/lettuce-core/wiki/Redis-URI-and-connection-details#uri-syntax[Redis URI syntax]. +| + +|`redisearch.default-schema-name` +|The schema that contains all tables defined without a qualifying schema name. +|`default` + +|`redisearch.username` +|Redis connection username. +| + +|`redisearch.password` +|Redis connection password. +| + +|`redisearch.cluster` +|Connect to a Redis Cluster. +|`false` -`redisearch.uri`:: A connection string containing the protocol, credential, and host info to connect to Redis. -+ -For example, the connection string may use the format `redis://:@:`. -+ -See the https://github.com/lettuce-io/lettuce-core/wiki/Redis-URI-and-connection-details#uri-syntax[Redis URI syntax] for more information. -+ -This property is required and has no default. A Redis URI must be provided to connect to RediSearch. +|`redisearch.case-insensitive-names` +|Match index names case insensitively. +|`false` -`redisearch.username`:: Redis connection username. +|`redisearch.default-limit` +|Max number of documents returned by FT.SEARCH and FT.AGGREGATE when no limit is present in the SQL query. +|`10000` -`redisearch.password`:: Redis connection password. +|`redisearch.cursor-count` +|Number of rows read during each https://redis.io/docs/stack/search/reference/aggregations/#cursor-api[aggregation cursor] fetch. +|`1000` +|=== -`redisearch.insecure`:: Allow insecure connections (e.g. invalid certificates) to Redis when using SSL. -+ -This property is optional and the default is `false`. +== TLS security +The RediSearch connector provides additional security options to support Redis servers with TLS mode. -`redisearch.timeout`:: Redis command timeout in seconds. -+ -This property is optional and the default is `60`. +The allowed configuration values are: -`redisearch.case-insensitive-name-matching`:: Match index names case insensitively. -+ -This property is optional and the default is `false`. +[cols="1,1"] +|=== +|Property name |Description -`redisearch.default-limit`:: Max number of documents returned by FT.SEARCH and FT.AGGREGATE when no limit is present in the SQL query. -+ -This property is optional and the default is `10000`. +|`redisearch.insecure` +|Allow insecure connections (e.g. invalid certificates) when using SSL. -`redisearch.cursor-count`:: Number of rows read during each https://redis.io/docs/stack/search/reference/aggregations/#cursor-api[aggregation cursor] fetch. -+ -This property is optional and if it is not specified the RediSearch default is used. +|`redisearch.cacert-path` +|X.509 CA certificate file to verify with. + +|`redisearch.key-path` +|PKCS#8 private key file to authenticate with (PEM format). + +|`redisearch.key-password` +|Password of the private key file, or null if it's not password-protected. + +|`redisearch.cert-path` +|X.509 certificate chain file to authenticate with (PEM format). +|=== == Docker Example -1. Launch the containers -+ +.Clone the project +[source,console,subs="verbatim,attributes"] +---- +git clone {project-url}.git +cd {project-name} +---- + +.Launch the containers [source,console] ---- docker-compose up ---- -2. Create a RediSearch index -+ +.Create a RediSearch index [source,console] ---- -docker exec redismod /usr/local/bin/redis-cli FT.CREATE beers ON HASH PREFIX 1 beer: SCHEMA id TAG SORTABLE brewery_id TAG SORTABLE name TEXT SORTABLE abv NUMERIC SORTABLE descript TEXT style_name TAG SORTABLE cat_name TAG SORTABLE +docker exec redis /usr/local/bin/redis-cli FT.CREATE beers ON HASH PREFIX 1 beer: SCHEMA id TAG SORTABLE brewery_id TAG SORTABLE name TEXT SORTABLE abv NUMERIC SORTABLE descript TEXT style_name TAG SORTABLE cat_name TAG SORTABLE ---- -3. Add documents to the index -+ +.Add documents to the index [source,console] ---- docker run --add-host=host.docker.internal:host-gateway jruaux/riot-file -h host.docker.internal import https://storage.googleapis.com/jrx/beers.json hset --keyspace beer --keys id ---- -4. Run the Trino CLI -+ +.Run the Trino CLI [source,console] ---- docker exec -it trino trino --catalog redisearch --schema default ---- -5. Run a SQL query -+ +.Run a SQL query [source,console] ---- trino:default> select * from beers; ---- +== Tableau + +Follow these steps to connect Tableau: https://help.tableau.com/current/pro/desktop/en-us/examples_presto.htm + == Install and Run -=== Install Trino +=== Trino + +Trino installation instructions are available here: https://trino.io/docs/current/installation.html + +The following steps deploy a single-node Trino server on Ubuntu. + +Trino requires a 64-bit of Java 17. +It is recommended to https://www.azul.com/downloads/?package=jdk[Azul Zulu] as the JDK. +[source,console] +---- +$ java -version +openjdk version "17.0.4.1" 2022-08-12 LTS +OpenJDK Runtime Environment Zulu17.36+17-CA (build 17.0.4.1+1-LTS) +OpenJDK 64-Bit Server VM Zulu17.36+17-CA (build 17.0.4.1+1-LTS, mixed mode, sharing) +---- + +Download the Trino server tarball and unpack it. +[source,console,subs="verbatim,attributes"] +---- +wget https://repo1.maven.org/maven2/io/trino/trino-server/{trino-version}/trino-server-{trino-version}.tar.gz +mkdir {trino-dir} +tar xzvf trino-server-{trino-version}.tar.gz --directory {trino-dir} --strip-components 1 +---- + +Trino needs a data directory for storing logs, etc. +It is recommended to create a data directory outside of the installation directory, which allows it to be easily preserved when upgrading Trino. +[source,console,subs="verbatim,attributes"] +---- +mkdir -p {trino-datadir} +---- + +Create an `etc` directory inside the installation directory to hold configuration files. +[source,console,subs="verbatim,attributes"] +---- +mkdir {trino-dir}/etc +---- + +Create a node properties file `etc/node.properties`. + +.{trino-dir}/etc/node.properties +[source,properties,subs="verbatim,attributes"] +---- +node.environment=production +node.id=ffffffff-ffff-ffff-ffff-ffffffffffff +node.data-dir={trino-datadir} +---- + +Create a JVM config file `etc/jvm.config`. + +.{trino-dir}/etc/jvm.config +[source,properties] +---- +-server +-Xmx16G +-XX:InitialRAMPercentage=80 +-XX:MaxRAMPercentage=80 +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+ExitOnOutOfMemoryError +-XX:+HeapDumpOnOutOfMemoryError +-XX:-OmitStackTraceInFastThrow +-XX:ReservedCodeCacheSize=512M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 +-XX:+UnlockDiagnosticVMOptions +-XX:+UseAESCTRIntrinsics +---- -https://trino.io/docs/current/installation.html +Create a config properties file `etc/config.properties`. +.{trino-dir}/etc/config.properties +[source,properties] +---- +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080 +---- -For macOS: `brew install trino` +Create a log levels file `etc/log.properties`. +.{trino-dir}/etc/log.properties +[source,properties] +---- +io.trino=INFO +---- -=== Install the connector +=== Redis SQL -1. Download latest https://github.com/redis-field-engineering/{project-name}/releases/latest[release]. +Download latest {project-url}/releases/latest[release] and unzip without any directory structure under `/plugin/redisearch`. -2. Unzip the release without any directory structure under `/plugin/redisearch`: -+ [source,console,subs="verbatim,attributes"] ---- -unzip -j trino-redisearch-{project-version}.zip -d /opt/trino/plugin/redisearch +wget {project-url}/releases/download/v{trino-version}/{artifact-id}-{trino-version}.zip +unzip -j {artifact-id}-{project-version}.zip -d {trino-dir}/plugin/redisearch ---- -3. Create a `redisearch.properties` file under `/etc/catalog` directory: -+ -.redisearch.properties +Create a `redisearch.properties` file under `{trino-dir}/etc/catalog` directory: + [source,properties] ---- connector.name=redisearch redisearch.uri=redis://localhost:6379 ---- +Change and/or add <> as needed. + === Run -1. Start Trino server: -+ -[source,console] +==== Trino Server + +Start the Trino server by running: + +[source,console,subs="verbatim,attributes"] ---- -trino-server start +{trino-dir}/bin/launcher run ---- -2. Connect to Trino using the CLI: -+ -[source,console] +==== Trino CLI + +Download https://repo1.maven.org/maven2/io/trino/trino-cli/{trino-version}/trino-cli-{trino-version}-executable.jar[trino-cli-{trino-version}-executable.jar], rename it to `trino`, make it executable with `chmod +x`, and run it to show the version of the CLI: + +[source,console,subs="verbatim,attributes"] ---- -trino --catalog redisearch --schema default +wget https://repo1.maven.org/maven2/io/trino/trino-cli/{trino-version}/trino-cli-{trino-version}-executable.jar +mv trino-cli-{trino-version}-executable.jar trino +chmod +x trino ---- -3. Run a SQL query: -+ +Connect to Trino using the CLI: + [source,console] ---- -trino:default> select count(*) from myIdx; +./trino --catalog redisearch --schema default ---- -== Tableau +Run a SQL query: -Follow these steps to connect Tableau: https://help.tableau.com/current/pro/desktop/en-us/examples_presto.htm +[source,console] +---- +trino:default> select * from mySearchIndex; +---- == Build -Run these commands to build the Trino connector for RediSearch from source: +Run these commands to build the Trino connector for RediSearch from source (requires Java 17+): -[source,console] +[source,console,subs="verbatim,attributes"] ---- -git clone https://github.com/redis-field-engineering/redis-sql.git -cd redis-sql +git clone {project-url}.git +cd {project-name} ./gradlew clean build ---- + diff --git a/pom.xml b/pom.xml index 6b606a6..a2a8382 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,12 @@ io.trino trino-root - 397 + 380 com.redis trino-redisearch - 0.2.6-SNAPSHOT + 0.2.5-380 Trino - RediSearch Connector trino-plugin diff --git a/src/main/java/com/redis/trino/RediSearchMetadata.java b/src/main/java/com/redis/trino/RediSearchMetadata.java index 58d1b09..a91517c 100644 --- a/src/main/java/com/redis/trino/RediSearchMetadata.java +++ b/src/main/java/com/redis/trino/RediSearchMetadata.java @@ -29,7 +29,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.airlift.slice.SliceUtf8.getCodePointAt; import static io.trino.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; -import static io.trino.spi.expression.StandardFunctions.LIKE_FUNCTION_NAME; import static java.util.Objects.requireNonNull; import java.util.ArrayList; @@ -80,6 +79,7 @@ import io.trino.spi.expression.Call; import io.trino.spi.expression.ConnectorExpression; import io.trino.spi.expression.Constant; +import io.trino.spi.expression.StandardFunctions; import io.trino.spi.expression.Variable; import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.TupleDomain; @@ -337,7 +337,7 @@ public Optional> applyFilter(C } protected static boolean isSupportedLikeCall(Call call) { - if (!LIKE_FUNCTION_NAME.equals(call.getFunctionName())) { + if (!StandardFunctions.LIKE_PATTERN_FUNCTION_NAME.equals(call.getFunctionName())) { return false; } diff --git a/src/main/java/com/redis/trino/RediSearchSplitManager.java b/src/main/java/com/redis/trino/RediSearchSplitManager.java index 100611d..910f9f3 100644 --- a/src/main/java/com/redis/trino/RediSearchSplitManager.java +++ b/src/main/java/com/redis/trino/RediSearchSplitManager.java @@ -33,7 +33,6 @@ import io.trino.spi.connector.ConnectorSplitSource; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.FixedSplitSource; @@ -48,7 +47,7 @@ public RediSearchSplitManager(RediSearchSession session) { @Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, - ConnectorTableHandle table, DynamicFilter dynamicFilter, Constraint constraint) { + ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy, DynamicFilter dynamicFilter) { RediSearchSplit split = new RediSearchSplit(addresses); return new FixedSplitSource(List.of(split)); }