From 811acb2fe77464f679a09226a03c1995694c51b4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 4 Jun 2024 10:45:28 +0200 Subject: [PATCH 01/77] CASSANDRA-19635: Configure Jenkins to run integration tests with C* 5.x patch by Lukasz Antoniak; reviewed by Bret McGuire for CASSANDRA-19635 --- Jenkinsfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d38b7c63849..4f1ef95d101 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -261,7 +261,7 @@ pipeline { '3.11', // Previous Apache CassandraⓇ '4.0', // Previous Apache CassandraⓇ '4.1', // Current Apache CassandraⓇ - '5.0', // Development Apache CassandraⓇ + '5.0-beta1', // Development Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Long Term Support DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise @@ -411,14 +411,14 @@ pipeline { triggers { // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ - # Every weeknight (Monday - Friday) around 2:00 AM - ### JDK8 tests against 2.1, 3.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 - H 2 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 - ### JDK11 tests against 3.11, 4.0 and DSE 6.8 - H 2 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.0 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 - # Every weekend (Sunday) around 12:00 PM noon - ### JDK14 tests against 3.11, 4.0 and DSE 6.8 - H 12 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.0 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.14 + # Every weekend (Saturday, Sunday) around 2:00 AM + ### JDK8 tests against 2.1, 3.0, 4.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 + H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 + # Every weeknight (Monday - Friday) around 12:00 PM noon + ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -452,8 +452,8 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ + values '3.11', // Latest stable Apache CassandraⓇ + '4.1', // Development Apache CassandraⓇ 'dse-6.8.30' // Current DataStax Enterprise } axis { @@ -565,7 +565,7 @@ pipeline { '3.11', // Previous Apache CassandraⓇ '4.0', // Previous Apache CassandraⓇ '4.1', // Current Apache CassandraⓇ - '5.0', // Development Apache CassandraⓇ + '5.0-beta1', // Development Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Last EOSL DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise From 85bb4065098b887d2dda26eb14423ce4fc687045 Mon Sep 17 00:00:00 2001 From: Brad Schoening Date: Tue, 4 Jun 2024 17:30:41 -0400 Subject: [PATCH 02/77] update badge URL to org.apache.cassandra/java-driver-core --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c53c8f2db29..2a30cb68c9a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ :warning: The java-driver has recently been donated by Datastax to The Apache Software Foundation and the Apache Cassandra project. Bear with us as we move assets and coordinates. -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.datastax.oss/java-driver-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.datastax.oss/java-driver-core) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core) *If you're reading this on github.com, please note that this is the readme for the development version and that some features described here might not yet have been released. You can find the From d0a1e44a4415c7a0489f8c35ee9ce49e20d7bc61 Mon Sep 17 00:00:00 2001 From: Benoit Tellier Date: Sun, 22 Jan 2023 13:58:19 +0700 Subject: [PATCH 03/77] Limit calls to Conversions.resolveExecutionProfile Those repeated calls account for a non-negligible portion of my application CPU (0.6%) and can definitly be a final field so that it gets resolved only once per CqlRequestHandler. patch by Benoit Tellier; reviewed by Andy Tolbert, and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1623 --- .../ContinuousRequestHandlerBase.java | 15 +++--- .../core/graph/GraphRequestHandler.java | 15 +++--- .../driver/internal/core/cql/Conversions.java | 38 +++++++++++-- .../internal/core/cql/CqlPrepareHandler.java | 11 ++-- .../internal/core/cql/CqlRequestHandler.java | 54 +++++++------------ 5 files changed, 75 insertions(+), 58 deletions(-) diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java index 44df3b3a03d..9a7be344721 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java @@ -648,12 +648,13 @@ public void operationComplete(@NonNull Future future) { } } else { LOG.trace("[{}] Request sent on {}", logPrefix, channel); - if (scheduleSpeculativeExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleSpeculativeExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = executionIndex + 1; // Note that `node` is the first node of the execution, it might not be the "slow" one // if there were retries, but in practice retries are rare. long nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, keyspace, statement, nextExecution); if (nextDelay >= 0) { scheduleSpeculativeExecution(nextExecution, nextDelay); @@ -787,12 +788,12 @@ public void onFailure(@NonNull Throwable error) { cancelTimeout(pageTimeout); LOG.trace(String.format("[%s] Request failure", logPrefix), error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { abort( @@ -945,7 +946,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { assert lock.isHeldByCurrentThread(); NodeMetricUpdater metricUpdater = ((DefaultNode) node).getMetricUpdater(); RetryVerdict verdict; - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; verdict = @@ -964,7 +965,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { DefaultNodeMetric.IGNORES_ON_READ_TIMEOUT); } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; - if (Conversions.resolveIdempotence(statement, context)) { + if (Conversions.resolveIdempotence(statement, executionProfile)) { verdict = retryPolicy.onWriteTimeoutVerdict( statement, @@ -999,7 +1000,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java index c2298458805..702da69b855 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java @@ -557,12 +557,13 @@ public void operationComplete(Future future) { cancel(); } else { inFlightCallbacks.add(this); - if (scheduleNextExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleNextExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = execution + 1; long nextDelay; try { nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, null, statement, nextExecution); } catch (Throwable cause) { // This is a bug in the policy, but not fatal since we have at least one other @@ -678,7 +679,7 @@ private void processErrorResponse(Error errorMessage) { trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); setFinalError(statement, error, node, execution); } else { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict; if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; @@ -699,7 +700,7 @@ private void processErrorResponse(Error errorMessage) { } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onWriteTimeoutVerdict( statement, writeTimeout.getConsistencyLevel(), @@ -731,7 +732,7 @@ private void processErrorResponse(Error errorMessage) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( @@ -810,12 +811,12 @@ public void onFailure(Throwable error) { } LOG.trace("[{}] Request failure, processing: {}", logPrefix, error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { setFinalError( diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java index 529664c6666..ff9384b3e24 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java @@ -535,29 +535,59 @@ public static CoordinatorException toThrowable( } } + /** Use {@link #resolveIdempotence(Request, DriverExecutionProfile)} instead. */ + @Deprecated public static boolean resolveIdempotence(Request request, InternalDriverContext context) { + return resolveIdempotence(request, resolveExecutionProfile(request, context)); + } + + public static boolean resolveIdempotence( + Request request, DriverExecutionProfile executionProfile) { Boolean requestIsIdempotent = request.isIdempotent(); - DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return (requestIsIdempotent == null) ? executionProfile.getBoolean(DefaultDriverOption.REQUEST_DEFAULT_IDEMPOTENCE) : requestIsIdempotent; } + /** Use {@link #resolveRequestTimeout(Request, DriverExecutionProfile)} instead. */ + @Deprecated public static Duration resolveRequestTimeout(Request request, InternalDriverContext context) { - DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); - return request.getTimeout() != null - ? request.getTimeout() + return resolveRequestTimeout(request, resolveExecutionProfile(request, context)); + } + + public static Duration resolveRequestTimeout( + Request request, DriverExecutionProfile executionProfile) { + Duration timeout = request.getTimeout(); + return timeout != null + ? timeout : executionProfile.getDuration(DefaultDriverOption.REQUEST_TIMEOUT); } + /** Use {@link #resolveRetryPolicy(InternalDriverContext, DriverExecutionProfile)} instead. */ + @Deprecated public static RetryPolicy resolveRetryPolicy(Request request, InternalDriverContext context) { DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return context.getRetryPolicy(executionProfile.getName()); } + public static RetryPolicy resolveRetryPolicy( + InternalDriverContext context, DriverExecutionProfile executionProfile) { + return context.getRetryPolicy(executionProfile.getName()); + } + + /** + * Use {@link #resolveSpeculativeExecutionPolicy(InternalDriverContext, DriverExecutionProfile)} + * instead. + */ + @Deprecated public static SpeculativeExecutionPolicy resolveSpeculativeExecutionPolicy( Request request, InternalDriverContext context) { DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return context.getSpeculativeExecutionPolicy(executionProfile.getName()); } + + public static SpeculativeExecutionPolicy resolveSpeculativeExecutionPolicy( + InternalDriverContext context, DriverExecutionProfile executionProfile) { + return context.getSpeculativeExecutionPolicy(executionProfile.getName()); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java index 6faa8eee59f..8fe1adb20b1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java @@ -92,6 +92,7 @@ public class CqlPrepareHandler implements Throttled { private final Timeout scheduledTimeout; private final RequestThrottler throttler; private final Boolean prepareOnAllNodes; + private final DriverExecutionProfile executionProfile; private volatile InitialPrepareCallback initialCallback; // The errors on the nodes that were already tried (lazily initialized on the first error). @@ -111,7 +112,7 @@ protected CqlPrepareHandler( this.initialRequest = request; this.session = session; this.context = context; - DriverExecutionProfile executionProfile = Conversions.resolveExecutionProfile(request, context); + executionProfile = Conversions.resolveExecutionProfile(request, context); this.queryPlan = context .getLoadBalancingPolicyWrapper() @@ -131,7 +132,7 @@ protected CqlPrepareHandler( }); this.timer = context.getNettyOptions().getTimer(); - Duration timeout = Conversions.resolveRequestTimeout(request, context); + Duration timeout = Conversions.resolveRequestTimeout(request, executionProfile); this.scheduledTimeout = scheduleTimeout(timeout); this.prepareOnAllNodes = executionProfile.getBoolean(DefaultDriverOption.PREPARE_ON_ALL_NODES); @@ -292,7 +293,7 @@ private CompletionStage prepareOnOtherNode(PrepareRequest request, Node no false, toPrepareMessage(request), request.getCustomPayload(), - Conversions.resolveRequestTimeout(request, context), + Conversions.resolveRequestTimeout(request, executionProfile), throttler, session.getMetricUpdater(), logPrefix); @@ -419,7 +420,7 @@ private void processErrorResponse(Error errorMessage) { } else { // Because prepare requests are known to always be idempotent, we call the retry policy // directly, without checking the flag. - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(request, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict = retryPolicy.onErrorResponseVerdict(request, error, retryCount); processRetryVerdict(verdict, error); } @@ -457,7 +458,7 @@ public void onFailure(Throwable error) { LOG.trace("[{}] Request failure, processing: {}", logPrefix, error.toString()); RetryVerdict verdict; try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(request, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(request, error, retryCount); } catch (Throwable cause) { setFinalError( diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index e7e334d57d8..a1c6b0e5466 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -126,6 +126,7 @@ public class CqlRequestHandler implements Throttled { private final RequestThrottler throttler; private final RequestTracker requestTracker; private final SessionMetricUpdater sessionMetricUpdater; + private final DriverExecutionProfile executionProfile; // The errors on the nodes that were already tried (lazily initialized on the first error). // We don't use a map because nodes can appear multiple times. @@ -167,7 +168,8 @@ protected CqlRequestHandler( this.sessionMetricUpdater = session.getMetricUpdater(); this.timer = context.getNettyOptions().getTimer(); - Duration timeout = Conversions.resolveRequestTimeout(statement, context); + this.executionProfile = Conversions.resolveExecutionProfile(initialStatement, context); + Duration timeout = Conversions.resolveRequestTimeout(statement, executionProfile); this.scheduledTimeout = scheduleTimeout(timeout); this.throttler = context.getRequestThrottler(); @@ -176,8 +178,6 @@ protected CqlRequestHandler( @Override public void onThrottleReady(boolean wasDelayed) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(initialStatement, context); if (wasDelayed // avoid call to nanoTime() if metric is disabled: && sessionMetricUpdater.isEnabled( @@ -276,8 +276,6 @@ private void sendRequest( retryCount, scheduleNextExecution, logPrefix); - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(statement, context); Message message = Conversions.toMessage(statement, executionProfile, context); channel .write(message, statement.isTracing(), statement.getCustomPayload(), nodeResponseCallback) @@ -336,37 +334,28 @@ private void setFinalResult( totalLatencyNanos = completionTimeNanos - startTimeNanos; long nodeLatencyNanos = completionTimeNanos - callback.nodeStartTimeNanos; requestTracker.onNodeSuccess( - callback.statement, - nodeLatencyNanos, - callback.executionProfile, - callback.node, - logPrefix); + callback.statement, nodeLatencyNanos, executionProfile, callback.node, logPrefix); requestTracker.onSuccess( - callback.statement, - totalLatencyNanos, - callback.executionProfile, - callback.node, - logPrefix); + callback.statement, totalLatencyNanos, executionProfile, callback.node, logPrefix); } if (sessionMetricUpdater.isEnabled( - DefaultSessionMetric.CQL_REQUESTS, callback.executionProfile.getName())) { + DefaultSessionMetric.CQL_REQUESTS, executionProfile.getName())) { if (completionTimeNanos == NANOTIME_NOT_MEASURED_YET) { completionTimeNanos = System.nanoTime(); totalLatencyNanos = completionTimeNanos - startTimeNanos; } sessionMetricUpdater.updateTimer( DefaultSessionMetric.CQL_REQUESTS, - callback.executionProfile.getName(), + executionProfile.getName(), totalLatencyNanos, TimeUnit.NANOSECONDS); } } // log the warnings if they have NOT been disabled if (!executionInfo.getWarnings().isEmpty() - && callback.executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOG_WARNINGS) + && executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOG_WARNINGS) && LOG.isWarnEnabled()) { - logServerWarnings( - callback.statement, callback.executionProfile, executionInfo.getWarnings()); + logServerWarnings(callback.statement, executionProfile, executionInfo.getWarnings()); } } catch (Throwable error) { setFinalError(callback.statement, error, callback.node, -1); @@ -418,21 +407,17 @@ private ExecutionInfo buildExecutionInfo( schemaInAgreement, session, context, - callback.executionProfile); + executionProfile); } @Override public void onThrottleFailure(@NonNull RequestThrottlingException error) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(initialStatement, context); sessionMetricUpdater.incrementCounter( DefaultSessionMetric.THROTTLING_ERRORS, executionProfile.getName()); setFinalError(initialStatement, error, null, -1); } private void setFinalError(Statement statement, Throwable error, Node node, int execution) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(statement, context); if (error instanceof DriverException) { ((DriverException) error) .setExecutionInfo( @@ -475,7 +460,6 @@ private class NodeResponseCallback private final long nodeStartTimeNanos = System.nanoTime(); private final Statement statement; - private final DriverExecutionProfile executionProfile; private final Node node; private final Queue queryPlan; private final DriverChannel channel; @@ -505,7 +489,6 @@ private NodeResponseCallback( this.retryCount = retryCount; this.scheduleNextExecution = scheduleNextExecution; this.logPrefix = logPrefix + "|" + execution; - this.executionProfile = Conversions.resolveExecutionProfile(statement, context); } // this gets invoked once the write completes. @@ -544,12 +527,13 @@ public void operationComplete(Future future) throws Exception { cancel(); } else { inFlightCallbacks.add(this); - if (scheduleNextExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleNextExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = execution + 1; long nextDelay; try { nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, keyspace, statement, nextExecution); } catch (Throwable cause) { // This is a bug in the policy, but not fatal since we have at least one other @@ -697,7 +681,7 @@ private void processErrorResponse(Error errorMessage) { true, reprepareMessage, repreparePayload.customPayload, - Conversions.resolveRequestTimeout(statement, context), + Conversions.resolveRequestTimeout(statement, executionProfile), throttler, sessionMetricUpdater, logPrefix); @@ -767,7 +751,7 @@ private void processErrorResponse(Error errorMessage) { trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); setFinalError(statement, error, node, execution); } else { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict; if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; @@ -788,7 +772,7 @@ private void processErrorResponse(Error errorMessage) { } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onWriteTimeoutVerdict( statement, writeTimeout.getConsistencyLevel(), @@ -820,7 +804,7 @@ private void processErrorResponse(Error errorMessage) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( @@ -899,12 +883,12 @@ public void onFailure(Throwable error) { } LOG.trace("[{}] Request failure, processing: {}", logPrefix, error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { setFinalError( From a17f7be614a09ab81bc2982b7f7ab3a123b4ab28 Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Thu, 22 Aug 2024 14:28:46 +0200 Subject: [PATCH 04/77] autolink JIRA tickets in commit messages patch by Stefan Miklosovic; reviewed by Michael Semb Wever for CASSANDRA-19854 --- .asf.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 5ebca4b6e33..ad58f536398 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,6 +19,7 @@ notifications: commits: commits@cassandra.apache.org issues: commits@cassandra.apache.org pullrequests: pr@cassandra.apache.org + jira_options: link worklog github: description: "Java Driver for Apache Cassandra®" @@ -31,6 +32,5 @@ github: wiki: false issues: false projects: false - -notifications: - jira_options: link worklog + autolink_jira: + - CASSANDRA From 0962794b2ec724d9939cd47380e68b979b46f693 Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Sun, 20 Nov 2022 19:14:07 -0800 Subject: [PATCH 05/77] Don't return empty routing key when partition key is unbound DefaultBoundStatement#getRoutingKey has logic to infer the routing key when no one has explicitly called setRoutingKey or otherwise set the routing key on the statement. It however doesn't check for cases where nothing has been bound yet on the statement. This causes more problems if the user decides to get a BoundStatementBuilder from the PreparedStatement, set some fields on it, and then copy it by constructing new BoundStatementBuilder objects with the BoundStatement as a parameter, since the empty ByteBuffer gets copied to all bound statements, resulting in all requests being targeted to the same Cassandra node in a token-aware load balancing policy. patch by Ammar Khaku; reviewed by Andy Tolbert, and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1620 --- .../core/cql/DefaultBoundStatement.java | 3 ++- .../driver/core/cql/PreparedStatementIT.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java index fb6b8fd7b27..3cf99c1be6e 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java @@ -360,7 +360,8 @@ public ByteBuffer getRoutingKey() { if (indices.isEmpty()) { return null; } else if (indices.size() == 1) { - return getBytesUnsafe(indices.get(0)); + int index = indices.get(0); + return isSet(index) ? getBytesUnsafe(index) : null; } else { ByteBuffer[] components = new ByteBuffer[indices.size()]; for (int i = 0; i < components.length; i++) { diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java index c0df01e3519..5671a7684e5 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java @@ -527,6 +527,25 @@ private void should_infer_routing_information_when_partition_key_is_bound(String assertThat(tokenFactory.hash(boundStatement.getRoutingKey())).isEqualTo(expectedToken); } + @Test + public void should_return_null_routing_information_when_single_partition_key_is_unbound() { + should_return_null_routing_information_when_single_partition_key_is_unbound( + "SELECT a FROM prepared_statement_test WHERE a = ?"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "INSERT INTO prepared_statement_test (a) VALUES (?)"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "UPDATE prepared_statement_test SET b = 1 WHERE a = ?"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "DELETE FROM prepared_statement_test WHERE a = ?"); + } + + private void should_return_null_routing_information_when_single_partition_key_is_unbound( + String queryString) { + CqlSession session = sessionRule.session(); + BoundStatement boundStatement = session.prepare(queryString).bind(); + assertThat(boundStatement.getRoutingKey()).isNull(); + } + private static Iterable firstPageOf(CompletionStage stage) { return CompletableFutures.getUninterruptibly(stage).currentPage(); } From e6ae1933667066bf16ac3ac5203a2a6fdadd1946 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 16 Aug 2024 11:06:09 +0200 Subject: [PATCH 06/77] JAVA-3167: CompletableFutures.allSuccessful() may return never completed future patch by Lukasz Antoniak; reviewed by Andy Tolbert, and Bret McGuire for JAVA-3167 --- .../util/concurrent/CompletableFutures.java | 5 +++- .../concurrent/CompletableFuturesTest.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java index 03265bd1d77..275b2ddfeef 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java @@ -100,7 +100,10 @@ public static CompletionStage allSuccessful(List> i } else { Throwable finalError = errors.get(0); for (int i = 1; i < errors.size(); i++) { - finalError.addSuppressed(errors.get(i)); + Throwable suppressedError = errors.get(i); + if (finalError != suppressedError) { + finalError.addSuppressed(suppressedError); + } } result.completeExceptionally(finalError); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java new file mode 100644 index 00000000000..8a710e02d50 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java @@ -0,0 +1,29 @@ +package com.datastax.oss.driver.internal.core.util.concurrent; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class CompletableFuturesTest { + @Test + public void should_not_suppress_identical_exceptions() throws Exception { + RuntimeException error = new RuntimeException(); + CompletableFuture future1 = new CompletableFuture<>(); + future1.completeExceptionally(error); + CompletableFuture future2 = new CompletableFuture<>(); + future2.completeExceptionally(error); + try { + // if timeout exception is thrown, it indicates that CompletableFutures.allSuccessful() + // did not complete the returned future and potentially caller will wait infinitely + CompletableFutures.allSuccessful(Arrays.asList(future1, future2)) + .toCompletableFuture() + .get(1, TimeUnit.SECONDS); + } catch (ExecutionException e) { + assertThat(e.getCause()).isEqualTo(error); + } + } +} From 5ee12acfff720047db7611a6f54450c1646031a3 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 3 Sep 2024 16:06:04 -0500 Subject: [PATCH 07/77] ninja-fix Various test fixes --- .../concurrent/CompletableFuturesTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java index 8a710e02d50..04f96f185fd 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java @@ -1,6 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.datastax.oss.driver.internal.core.util.concurrent; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; import java.util.Arrays; import java.util.concurrent.CompletableFuture; @@ -22,6 +40,7 @@ public void should_not_suppress_identical_exceptions() throws Exception { CompletableFutures.allSuccessful(Arrays.asList(future1, future2)) .toCompletableFuture() .get(1, TimeUnit.SECONDS); + fail(); } catch (ExecutionException e) { assertThat(e.getCause()).isEqualTo(error); } From 9cfb4f6712e392c1b6c87db268565fd3b27d0c5c Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Thu, 5 Sep 2024 13:04:52 +0200 Subject: [PATCH 08/77] Run integration tests with DSE 6.9.0 patch by Lukasz Antoniak; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1955 --- Jenkinsfile | 19 +++++++++++++------ .../datastax/oss/driver/api/core/Version.java | 1 + .../driver/api/testinfra/ccm/CcmBridge.java | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4f1ef95d101..4cc20d79604 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -268,6 +268,7 @@ pipeline { 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise 'ALL'], description: '''Apache Cassandra® and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS builds @@ -325,6 +326,10 @@ pipeline { + + + +
dse-6.8.30 DataStax Enterprise v6.8.x
dse-6.9.0DataStax Enterprise v6.9.x
''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_JABBA_VERSION', @@ -416,9 +421,9 @@ pipeline { H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -452,9 +457,10 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ - 'dse-6.8.30' // Current DataStax Enterprise + values '3.11', // Latest stable Apache CassandraⓇ + '4.1', // Development Apache CassandraⓇ + 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'JABBA_VERSION' @@ -571,7 +577,8 @@ pipeline { 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise - 'dse-6.8.30' // Current DataStax Enterprise + 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } } when { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java index 3f12c54faf7..4de006da268 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java @@ -56,6 +56,7 @@ public class Version implements Comparable, Serializable { @NonNull public static final Version V5_0_0 = Objects.requireNonNull(parse("5.0.0")); @NonNull public static final Version V6_7_0 = Objects.requireNonNull(parse("6.7.0")); @NonNull public static final Version V6_8_0 = Objects.requireNonNull(parse("6.8.0")); + @NonNull public static final Version V6_9_0 = Objects.requireNonNull(parse("6.9.0")); private final int major; private final int minor; diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 995513e3919..5b0c114a5fe 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -479,7 +479,8 @@ private Optional overrideJvmVersionForDseWorkloads() { return Optional.empty(); } - if (getDseVersion().get().compareTo(Version.parse("6.8.19")) < 0) { + if (getDseVersion().get().compareTo(Version.V6_9_0) >= 0) { + // DSE 6.9.0 supports only JVM 11 onwards (also with graph workload) return Optional.empty(); } From c961012000efffd3d50476d4549487f7fc538c01 Mon Sep 17 00:00:00 2001 From: Henry Hughes Date: Wed, 23 Aug 2023 15:35:42 -0700 Subject: [PATCH 09/77] JAVA-3117: Call CcmCustomRule#after if CcmCustomRule#before fails to allow subsequent tests to run patch by Henry Hughes; reviewed by Alexandre Dutra and Andy Tolbert for JAVA-3117 --- .../api/testinfra/ccm/CustomCcmRule.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java index 58bafd438f8..cf150b12f55 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java @@ -18,6 +18,8 @@ package com.datastax.oss.driver.api.testinfra.ccm; import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A rule that creates a ccm cluster that can be used in a test. This should be used if you plan on @@ -30,6 +32,7 @@ */ public class CustomCcmRule extends BaseCcmRule { + private static final Logger LOG = LoggerFactory.getLogger(CustomCcmRule.class); private static final AtomicReference CURRENT = new AtomicReference<>(); CustomCcmRule(CcmBridge ccmBridge) { @@ -39,7 +42,20 @@ public class CustomCcmRule extends BaseCcmRule { @Override protected void before() { if (CURRENT.get() == null && CURRENT.compareAndSet(null, this)) { - super.before(); + try { + super.before(); + } catch (Exception e) { + // ExternalResource will not call after() when before() throws an exception + // Let's try and clean up and release the lock we have in CURRENT + LOG.warn( + "Error in CustomCcmRule before() method, attempting to clean up leftover state", e); + try { + after(); + } catch (Exception e1) { + LOG.warn("Error cleaning up CustomCcmRule before() failure", e1); + } + throw e; + } } else if (CURRENT.get() != this) { throw new IllegalStateException( "Attempting to use a Ccm rule while another is in use. This is disallowed"); From 77805f5103354cadb360384f4f41e0eca73d72f4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 2 Sep 2024 06:44:53 +0200 Subject: [PATCH 10/77] JAVA-3149: Support request cancellation in request throttler patch by Lukasz Antoniak; reviewed by Andy Tolbert and Chris Lohfink for JAVA-3149 --- .../ContinuousRequestHandlerBase.java | 1 + .../core/graph/GraphRequestHandler.java | 1 + .../session/throttling/RequestThrottler.java | 8 +++++ .../internal/core/cql/CqlPrepareHandler.java | 1 + .../internal/core/cql/CqlRequestHandler.java | 1 + .../ConcurrencyLimitingRequestThrottler.java | 16 +++++++++ .../PassThroughRequestThrottler.java | 5 +++ .../RateLimitingRequestThrottler.java | 12 +++++++ ...ncurrencyLimitingRequestThrottlerTest.java | 5 +++ .../RateLimitingRequestThrottlerTest.java | 13 ++++++- .../driver/core/throttling/ThrottlingIT.java | 34 +++++++++++++++---- 11 files changed, 89 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java index 9a7be344721..0453022cb6a 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java @@ -410,6 +410,7 @@ public void cancel() { cancelScheduledTasks(null); cancelGlobalTimeout(); + throttler.signalCancel(this); } private void cancelGlobalTimeout() { diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java index 702da69b855..5c9ceb00df2 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java @@ -153,6 +153,7 @@ public class GraphRequestHandler implements Throttled { try { if (t instanceof CancellationException) { cancelScheduledTasks(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java index cb55fac336c..7e2b41ebbdb 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java @@ -56,4 +56,12 @@ public interface RequestThrottler extends Closeable { * perform time-based eviction on pending requests. */ void signalTimeout(@NonNull Throttled request); + + /** + * Signals that a request has been cancelled. This indicates to the throttler that another request + * might be started. + */ + default void signalCancel(@NonNull Throttled request) { + // no-op for backward compatibility purposes + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java index 8fe1adb20b1..1ee1f303ab2 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java @@ -124,6 +124,7 @@ protected CqlPrepareHandler( try { if (t instanceof CancellationException) { cancelTimeout(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index a1c6b0e5466..0808bdce63f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -152,6 +152,7 @@ protected CqlRequestHandler( try { if (t instanceof CancellationException) { cancelScheduledTasks(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java index e8f27467c6f..438bed0953b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java @@ -145,6 +145,22 @@ public void signalTimeout(@NonNull Throttled request) { } } + @Override + public void signalCancel(@NonNull Throttled request) { + lock.lock(); + try { + if (!closed) { + if (queue.remove(request)) { // The request has been cancelled before it was active + LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); + } else { + onRequestDone(); + } + } + } finally { + lock.unlock(); + } + } + @SuppressWarnings("GuardedBy") // this method is only called with the lock held private void onRequestDone() { assert lock.isHeldByCurrentThread(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java index 714c712a4e8..2210e4b26f1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java @@ -69,6 +69,11 @@ public void signalTimeout(@NonNull Throttled request) { // nothing to do } + @Override + public void signalCancel(@NonNull Throttled request) { + // nothing to do + } + @Override public void close() throws IOException { // nothing to do diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java index 6536804ffee..03a693dc0fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java @@ -198,6 +198,18 @@ public void signalTimeout(@NonNull Throttled request) { } } + @Override + public void signalCancel(@NonNull Throttled request) { + lock.lock(); + try { + if (!closed && queue.remove(request)) { // The request has been cancelled before it was active + LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); + } + } finally { + lock.unlock(); + } + } + @Override public void close() { lock.lock(); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java index b587ac3daa2..c01b26c1e9f 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java @@ -88,6 +88,11 @@ public void should_allow_new_request_when_active_one_times_out() { should_allow_new_request_when_active_one_completes(throttler::signalTimeout); } + @Test + public void should_allow_new_request_when_active_one_canceled() { + should_allow_new_request_when_active_one_completes(throttler::signalCancel); + } + private void should_allow_new_request_when_active_one_completes( Consumer completeCallback) { // Given diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java index 7336fb447b6..0e0fe7c1c65 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.util.concurrent.ScheduledTaskCapturingEventLoop; @@ -33,6 +34,7 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -164,6 +166,15 @@ public void should_reject_when_queue_is_full() { @Test public void should_remove_timed_out_request_from_queue() { + testRemoveInvalidEventFromQueue(throttler::signalTimeout); + } + + @Test + public void should_remove_cancel_request_from_queue() { + testRemoveInvalidEventFromQueue(throttler::signalCancel); + } + + private void testRemoveInvalidEventFromQueue(Consumer completeCallback) { // Given for (int i = 0; i < 5; i++) { throttler.register(new MockThrottled()); @@ -174,7 +185,7 @@ public void should_remove_timed_out_request_from_queue() { throttler.register(queued2); // When - throttler.signalTimeout(queued1); + completeCallback.accept(queued1); // Then assertThatStage(queued2.started).isNotDone(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java index a6e7295eb09..6fa1a37355b 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java @@ -24,13 +24,16 @@ import com.datastax.oss.driver.api.core.RequestThrottlingException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.session.throttling.ConcurrencyLimitingRequestThrottler; import com.datastax.oss.simulacron.common.cluster.ClusterSpec; import com.datastax.oss.simulacron.common.stubbing.PrimeDsl; +import java.util.concurrent.CompletionStage; import java.util.concurrent.TimeUnit; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -39,21 +42,20 @@ public class ThrottlingIT { private static final String QUERY = "select * from foo"; + private static final int maxConcurrentRequests = 10; + private static final int maxQueueSize = 10; @Rule public SimulacronRule simulacron = new SimulacronRule(ClusterSpec.builder().withNodes(1)); - @Test - public void should_reject_request_when_throttling_by_concurrency() { + private DriverConfigLoader loader = null; + @Before + public void setUp() { // Add a delay so that requests don't complete during the test simulacron .cluster() .prime(PrimeDsl.when(QUERY).then(PrimeDsl.noRows()).delay(5, TimeUnit.SECONDS)); - - int maxConcurrentRequests = 10; - int maxQueueSize = 10; - - DriverConfigLoader loader = + loader = SessionUtils.configLoaderBuilder() .withClass( DefaultDriverOption.REQUEST_THROTTLER_CLASS, @@ -63,7 +65,10 @@ public void should_reject_request_when_throttling_by_concurrency() { maxConcurrentRequests) .withInt(DefaultDriverOption.REQUEST_THROTTLER_MAX_QUEUE_SIZE, maxQueueSize) .build(); + } + @Test + public void should_reject_request_when_throttling_by_concurrency() { try (CqlSession session = SessionUtils.newSession(simulacron, loader)) { // Saturate the session and fill the queue @@ -81,4 +86,19 @@ public void should_reject_request_when_throttling_by_concurrency() { + "(concurrent requests: 10, queue size: 10)"); } } + + @Test + public void should_propagate_cancel_to_throttler() { + try (CqlSession session = SessionUtils.newSession(simulacron, loader)) { + + // Try to saturate the session and fill the queue + for (int i = 0; i < maxConcurrentRequests + maxQueueSize; i++) { + CompletionStage future = session.executeAsync(QUERY); + future.toCompletableFuture().cancel(true); + } + + // The next query should be successful, because the previous queries were cancelled + session.execute(QUERY); + } + } } From 4fb51081a3a71b2017ddc3f43a7945e3a9d19e25 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 10 Jun 2024 12:42:29 +0200 Subject: [PATCH 11/77] Fix C* 3.0 tests failing on Jenkins patch by Lukasz Antoniak; reviewed by Bret McGuire reference: #1939 --- .../test/java/com/datastax/oss/driver/mapper/DeleteIT.java | 3 +-- .../com/datastax/oss/driver/mapper/InventoryITBase.java | 6 +++++- .../oss/driver/mapper/SelectCustomWhereClauseIT.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java index 0acdbeae53a..03e3597501c 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java @@ -60,8 +60,7 @@ description = ">= in WHERE clause not supported in legacy versions") public class DeleteIT extends InventoryITBase { - private static CustomCcmRule CCM_RULE = - CustomCcmRule.builder().withCassandraConfiguration("enable_sasi_indexes", "true").build(); + private static CustomCcmRule CCM_RULE = CustomCcmRule.builder().build(); private static final SessionRule SESSION_RULE = SessionRule.builder(CCM_RULE).build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java index 2be025b3739..9495003ae49 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java @@ -59,6 +59,10 @@ public abstract class InventoryITBase { new ProductSale(MP3_DOWNLOAD.getId(), DATE_3, 7, Uuids.startOf(915192000), 0.99, 12); protected static List createStatements(BaseCcmRule ccmRule) { + return createStatements(ccmRule, false); + } + + protected static List createStatements(BaseCcmRule ccmRule, boolean requiresSasiIndex) { ImmutableList.Builder builder = ImmutableList.builder() .add( @@ -71,7 +75,7 @@ protected static List createStatements(BaseCcmRule ccmRule) { "CREATE TABLE product_sale(id uuid, day text, ts uuid, customer_id int, price " + "double, count int, PRIMARY KEY ((id, day), customer_id, ts))"); - if (supportsSASI(ccmRule) && !isSasiBroken(ccmRule)) { + if (requiresSasiIndex && supportsSASI(ccmRule) && !isSasiBroken(ccmRule)) { builder.add( "CREATE CUSTOM INDEX product_description ON product(description) " + "USING 'org.apache.cassandra.index.sasi.SASIIndex' " diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java index 3df1ccd21a7..1f1b92b8623 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java @@ -75,7 +75,7 @@ public static void setup() { SchemaChangeSynchronizer.withLock( () -> { - for (String query : createStatements(CCM_RULE)) { + for (String query : createStatements(CCM_RULE, true)) { session.execute( SimpleStatement.builder(query) .setExecutionProfile(SESSION_RULE.slowProfile()) From 6d3ba47631ebde78460168a2d33c4facde0bd731 Mon Sep 17 00:00:00 2001 From: Jason Koch Date: Mon, 12 Aug 2024 22:52:13 -0700 Subject: [PATCH 12/77] Reduce lock held duration in ConcurrencyLimitingRequestThrottler It might take some (small) time for callback handling when the throttler request proceeds to submission. Before this change, the throttler proceed request will happen while holding the lock, preventing other tasks from proceeding when there is spare capacity and even preventing tasks from enqueuing until the callback completes. By tracking the expected outcome, we can perform the callback outside of the lock. This means that request registration and submission can proceed even when a long callback is being processed. patch by Jason Koch; Reviewed by Andy Tolbert and Chris Lohfink for CASSANDRA-19922 --- .../ConcurrencyLimitingRequestThrottler.java | 39 ++++- ...ncurrencyLimitingRequestThrottlerTest.java | 143 ++++++++++++++++-- .../session/throttling/MockThrottled.java | 30 +++- .../RateLimitingRequestThrottlerTest.java | 30 ++-- 4 files changed, 206 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java index 438bed0953b..ffe0ffe9650 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; import java.util.concurrent.locks.ReentrantLock; @@ -87,6 +88,8 @@ public ConcurrencyLimitingRequestThrottler(DriverContext context) { @Override public void register(@NonNull Throttled request) { + boolean notifyReadyRequired = false; + lock.lock(); try { if (closed) { @@ -96,7 +99,7 @@ public void register(@NonNull Throttled request) { // We have capacity for one more concurrent request LOG.trace("[{}] Starting newly registered request", logPrefix); concurrentRequests += 1; - request.onThrottleReady(false); + notifyReadyRequired = true; } else if (queue.size() < maxQueueSize) { LOG.trace("[{}] Enqueuing request", logPrefix); queue.add(request); @@ -112,16 +115,26 @@ public void register(@NonNull Throttled request) { } finally { lock.unlock(); } + + // no need to hold the lock while allowing the task to progress + if (notifyReadyRequired) { + request.onThrottleReady(false); + } } @Override public void signalSuccess(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @Override @@ -131,48 +144,62 @@ public void signalError(@NonNull Throttled request, @NonNull Throwable error) { @Override public void signalTimeout(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { if (!closed) { if (queue.remove(request)) { // The request timed out before it was active LOG.trace("[{}] Removing timed out request from the queue", logPrefix); } else { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } } } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @Override public void signalCancel(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { if (!closed) { if (queue.remove(request)) { // The request has been cancelled before it was active LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); } else { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } } } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @SuppressWarnings("GuardedBy") // this method is only called with the lock held - private void onRequestDone() { + @Nullable + private Throttled onRequestDoneAndDequeNext() { assert lock.isHeldByCurrentThread(); if (!closed) { if (queue.isEmpty()) { concurrentRequests -= 1; } else { LOG.trace("[{}] Starting dequeued request", logPrefix); - queue.poll().onThrottleReady(true); // don't touch concurrentRequests since we finished one but started another + return queue.poll(); } } + + // no next task was dequeued + return null; } @Override diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java index c01b26c1e9f..7eb682070cd 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.function.Consumer; import org.junit.Before; import org.junit.Test; @@ -67,7 +68,7 @@ public void should_start_immediately_when_under_capacity() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getConcurrentRequests()).isEqualTo(1); assertThat(throttler.getQueue()).isEmpty(); } @@ -98,7 +99,7 @@ private void should_allow_new_request_when_active_one_completes( // Given MockThrottled first = new MockThrottled(); throttler.register(first); - assertThatStage(first.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); for (int i = 0; i < 4; i++) { // fill to capacity throttler.register(new MockThrottled()); } @@ -113,7 +114,7 @@ private void should_allow_new_request_when_active_one_completes( throttler.register(incoming); // Then - assertThatStage(incoming.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(incoming.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).isEmpty(); } @@ -132,7 +133,7 @@ public void should_enqueue_when_over_capacity() { throttler.register(incoming); // Then - assertThatStage(incoming.started).isNotDone(); + assertThatStage(incoming.ended).isNotDone(); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).containsExactly(incoming); } @@ -157,20 +158,20 @@ private void should_dequeue_when_active_completes(Consumer completeCa // Given MockThrottled first = new MockThrottled(); throttler.register(first); - assertThatStage(first.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); for (int i = 0; i < 4; i++) { throttler.register(new MockThrottled()); } MockThrottled incoming = new MockThrottled(); throttler.register(incoming); - assertThatStage(incoming.started).isNotDone(); + assertThatStage(incoming.ended).isNotDone(); // When completeCallback.accept(first); // Then - assertThatStage(incoming.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(incoming.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).isEmpty(); } @@ -189,7 +190,7 @@ public void should_reject_when_queue_is_full() { throttler.register(incoming); // Then - assertThatStage(incoming.started) + assertThatStage(incoming.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -208,7 +209,7 @@ public void should_remove_timed_out_request_from_queue() { throttler.signalTimeout(queued1); // Then - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).hasSize(1); } @@ -223,7 +224,7 @@ public void should_reject_enqueued_when_closing() { for (int i = 0; i < 10; i++) { MockThrottled request = new MockThrottled(); throttler.register(request); - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); enqueued.add(request); } @@ -232,7 +233,7 @@ public void should_reject_enqueued_when_closing() { // Then for (MockThrottled request : enqueued) { - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -241,7 +242,125 @@ public void should_reject_enqueued_when_closing() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } + + @Test + public void should_run_throttle_callbacks_concurrently() throws InterruptedException { + // Given + + // a task is enqueued, which when in onThrottleReady, will stall latch countDown()ed + // register() should automatically start onThrottleReady on same thread + + // start a parallel thread + CountDownLatch firstRelease = new CountDownLatch(1); + MockThrottled first = new MockThrottled(firstRelease); + Runnable r = + () -> { + throttler.register(first); + first.ended.toCompletableFuture().thenRun(() -> throttler.signalSuccess(first)); + }; + Thread t = new Thread(r); + t.start(); + + // wait for the registration threads to reach await state + assertThatStage(first.started).isSuccess(); + assertThatStage(first.ended).isNotDone(); + + // When + // we concurrently submit a second shorter task + MockThrottled second = new MockThrottled(); + // (on a second thread, so that we can join and force a timeout in case + // registration is delayed) + Thread t2 = new Thread(() -> throttler.register(second)); + t2.start(); + t2.join(1_000); + + // Then + // registration will trigger callback, should complete ~immediately + assertThatStage(second.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + // first should still be unfinished + assertThatStage(first.started).isDone(); + assertThatStage(first.ended).isNotDone(); + // now finish, and verify + firstRelease.countDown(); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + + t.join(1_000); + } + + @Test + public void should_enqueue_tasks_quickly_when_callbacks_blocked() throws InterruptedException { + // Given + + // Multiple tasks are registered, up to the limit, and proceed into their + // callback + + // start five parallel threads + final int THREADS = 5; + Thread[] threads = new Thread[THREADS]; + CountDownLatch[] latches = new CountDownLatch[THREADS]; + MockThrottled[] throttled = new MockThrottled[THREADS]; + for (int i = 0; i < threads.length; i++) { + latches[i] = new CountDownLatch(1); + final MockThrottled itThrottled = new MockThrottled(latches[i]); + throttled[i] = itThrottled; + threads[i] = + new Thread( + () -> { + throttler.register(itThrottled); + itThrottled + .ended + .toCompletableFuture() + .thenRun(() -> throttler.signalSuccess(itThrottled)); + }); + threads[i].start(); + } + + // wait for the registration threads to be launched + // they are all waiting now + for (int i = 0; i < throttled.length; i++) { + assertThatStage(throttled[i].started).isSuccess(); + assertThatStage(throttled[i].ended).isNotDone(); + } + + // When + // we concurrently submit another task + MockThrottled last = new MockThrottled(); + throttler.register(last); + + // Then + // registration will enqueue the callback, and it should not + // take any time to proceed (ie: we should not be blocked) + // and there should be an element in the queue + assertThatStage(last.started).isNotDone(); + assertThatStage(last.ended).isNotDone(); + assertThat(throttler.getQueue()).containsExactly(last); + + // we still have not released, so old throttled threads should be waiting + for (int i = 0; i < throttled.length; i++) { + assertThatStage(throttled[i].started).isDone(); + assertThatStage(throttled[i].ended).isNotDone(); + } + + // now let us release .. + for (int i = 0; i < latches.length; i++) { + latches[i].countDown(); + } + + // .. and check everything finished up OK + for (int i = 0; i < latches.length; i++) { + assertThatStage(throttled[i].started).isSuccess(); + assertThatStage(throttled[i].ended).isSuccess(); + } + + // for good measure, we will also wait for the enqueued to complete + assertThatStage(last.started).isSuccess(); + assertThatStage(last.ended).isSuccess(); + + for (int i = 0; i < threads.length; i++) { + threads[i].join(1_000); + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java index b7cd0ee8a54..9e54e3d511f 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java @@ -19,21 +19,45 @@ import com.datastax.oss.driver.api.core.RequestThrottlingException; import com.datastax.oss.driver.api.core.session.throttling.Throttled; +import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; class MockThrottled implements Throttled { + final CompletionStage started = new CompletableFuture<>(); + final CompletionStage ended = new CompletableFuture<>(); + final CountDownLatch canRelease; - final CompletionStage started = new CompletableFuture<>(); + public MockThrottled() { + this(new CountDownLatch(0)); + } + + /* + * The releaseLatch can be provided to add some delay before the + * task readiness/fail callbacks complete. This can be used, eg, to + * imitate a slow callback. + */ + public MockThrottled(CountDownLatch releaseLatch) { + this.canRelease = releaseLatch; + } @Override public void onThrottleReady(boolean wasDelayed) { - started.toCompletableFuture().complete(wasDelayed); + started.toCompletableFuture().complete(null); + awaitRelease(); + ended.toCompletableFuture().complete(wasDelayed); } @Override public void onThrottleFailure(@NonNull RequestThrottlingException error) { - started.toCompletableFuture().completeExceptionally(error); + started.toCompletableFuture().complete(null); + awaitRelease(); + ended.toCompletableFuture().completeExceptionally(error); + } + + private void awaitRelease() { + Uninterruptibles.awaitUninterruptibly(canRelease); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java index 0e0fe7c1c65..1e15610bf7b 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java @@ -98,7 +98,7 @@ public void should_start_immediately_when_under_capacity() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getStoredPermits()).isEqualTo(4); assertThat(throttler.getQueue()).isEmpty(); } @@ -117,7 +117,7 @@ public void should_allow_new_request_when_under_rate() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).isEmpty(); } @@ -136,7 +136,7 @@ public void should_enqueue_when_over_rate() { throttler.register(request); // Then - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(request); @@ -160,7 +160,7 @@ public void should_reject_when_queue_is_full() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -188,7 +188,7 @@ private void testRemoveInvalidEventFromQueue(Consumer completeCallbac completeCallback.accept(queued1); // Then - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(queued2); } @@ -202,10 +202,10 @@ public void should_dequeue_when_draining_task_runs() { MockThrottled queued1 = new MockThrottled(); throttler.register(queued1); - assertThatStage(queued1.started).isNotDone(); + assertThatStage(queued1.ended).isNotDone(); MockThrottled queued2 = new MockThrottled(); throttler.register(queued2); - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).hasSize(2); @@ -230,8 +230,8 @@ public void should_dequeue_when_draining_task_runs() { task.run(); // Then - assertThatStage(queued1.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued1.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(queued2); // task reschedules itself since it did not empty the queue @@ -244,7 +244,7 @@ public void should_dequeue_when_draining_task_runs() { task.run(); // Then - assertThatStage(queued2.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued2.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).isEmpty(); assertThat(adminExecutor.nextTask()).isNull(); @@ -286,14 +286,14 @@ public void should_keep_accumulating_time_if_no_permits_created() { // Then MockThrottled queued = new MockThrottled(); throttler.register(queued); - assertThatStage(queued.started).isNotDone(); + assertThatStage(queued.ended).isNotDone(); // When clock.add(ONE_HUNDRED_MILLISECONDS); adminExecutor.nextTask().run(); // Then - assertThatStage(queued.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); } @Test @@ -306,7 +306,7 @@ public void should_reject_enqueued_when_closing() { for (int i = 0; i < 10; i++) { MockThrottled request = new MockThrottled(); throttler.register(request); - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); enqueued.add(request); } @@ -315,7 +315,7 @@ public void should_reject_enqueued_when_closing() { // Then for (MockThrottled request : enqueued) { - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -324,7 +324,7 @@ public void should_reject_enqueued_when_closing() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } } From 306bf3744a3d24ad01700e4e1d3c14b9a696927f Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Fri, 30 Sep 2022 15:36:36 -0700 Subject: [PATCH 13/77] Annotate BatchStatement, Statement, SimpleStatement methods with CheckReturnValue Since the driver's default implementation is for BatchStatement and SimpleStatement methods to be immutable, we should annotate those methods with @CheckReturnValue. Statement#setNowInSeconds implementations are immutable so annotate that too. patch by Ammar Khaku; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1607 --- core/revapi.json | 133 ++++++++++++++++++ .../driver/api/core/cql/BatchStatement.java | 8 ++ .../driver/api/core/cql/SimpleStatement.java | 7 + .../oss/driver/api/core/cql/Statement.java | 1 + 4 files changed, 149 insertions(+) diff --git a/core/revapi.json b/core/revapi.json index 318e29709ec..1c875895d6c 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -6956,6 +6956,139 @@ "old": "method java.lang.Throwable java.lang.Throwable::fillInStackTrace() @ com.fasterxml.jackson.databind.deser.UnresolvedForwardReference", "new": "method com.fasterxml.jackson.databind.deser.UnresolvedForwardReference com.fasterxml.jackson.databind.deser.UnresolvedForwardReference::fillInStackTrace()", "justification": "Upgrade jackson-databind to 2.13.4.1 to address CVEs, API change cause: https://github.com/FasterXML/jackson-databind/issues/3419" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BoundStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BoundStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.SimpleStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.SimpleStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int)", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::add(com.datastax.oss.driver.api.core.cql.BatchableStatement)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::add(com.datastax.oss.driver.api.core.cql.BatchableStatement)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(com.datastax.oss.driver.api.core.cql.BatchableStatement[])", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(com.datastax.oss.driver.api.core.cql.BatchableStatement[])", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(java.lang.Iterable>)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(java.lang.Iterable>)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::clear()", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::clear()", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setBatchType(com.datastax.oss.driver.api.core.cql.BatchType)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setBatchType(com.datastax.oss.driver.api.core.cql.BatchType)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "JAVA-2161: Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "JAVA-2161: Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setQuery(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setQuery(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setPositionalValues(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setPositionalValues(java.util.List)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValuesWithIds(java.util.Map)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValuesWithIds(java.util.Map)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" } ] } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java index 0f37ed71ce2..9deb33c6007 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java @@ -26,6 +26,7 @@ import com.datastax.oss.driver.internal.core.util.Sizes; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.PrimitiveSizes; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.ArrayList; @@ -164,6 +165,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement setBatchType(@NonNull BatchType newBatchType); /** @@ -180,6 +182,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * @see Request#getKeyspace() */ @NonNull + @CheckReturnValue BatchStatement setKeyspace(@Nullable CqlIdentifier newKeyspace); /** @@ -187,6 +190,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * setKeyspace(CqlIdentifier.fromCql(newKeyspaceName))}. */ @NonNull + @CheckReturnValue default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { return setKeyspace(CqlIdentifier.fromCql(newKeyspaceName)); } @@ -201,6 +205,7 @@ default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement add(@NonNull BatchableStatement statement); /** @@ -213,10 +218,12 @@ default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement addAll(@NonNull Iterable> statements); /** @see #addAll(Iterable) */ @NonNull + @CheckReturnValue default BatchStatement addAll(@NonNull BatchableStatement... statements) { return addAll(Arrays.asList(statements)); } @@ -231,6 +238,7 @@ default BatchStatement addAll(@NonNull BatchableStatement... statements) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement clear(); @Override diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java index fd5f456f11c..ef04cd14a5b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java @@ -28,6 +28,7 @@ import com.datastax.oss.protocol.internal.PrimitiveSizes; import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableList; import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.List; @@ -197,6 +198,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * @see #setNamedValuesWithIds(Map) */ @NonNull + @CheckReturnValue SimpleStatement setQuery(@NonNull String newQuery); /** @@ -209,6 +211,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * @see Request#getKeyspace() */ @NonNull + @CheckReturnValue SimpleStatement setKeyspace(@Nullable CqlIdentifier newKeyspace); /** @@ -216,6 +219,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * setKeyspace(CqlIdentifier.fromCql(newKeyspaceName))}. */ @NonNull + @CheckReturnValue default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { return setKeyspace(CqlIdentifier.fromCql(newKeyspaceName)); } @@ -236,6 +240,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * @see #setQuery(String) */ @NonNull + @CheckReturnValue SimpleStatement setPositionalValues(@NonNull List newPositionalValues); @NonNull @@ -256,6 +261,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * @see #setQuery(String) */ @NonNull + @CheckReturnValue SimpleStatement setNamedValuesWithIds(@NonNull Map newNamedValues); /** @@ -263,6 +269,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * converted on the fly with {@link CqlIdentifier#fromCql(String)}. */ @NonNull + @CheckReturnValue default SimpleStatement setNamedValues(@NonNull Map newNamedValues) { return setNamedValuesWithIds(DefaultSimpleStatement.wrapKeys(newNamedValues)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java index 594c627e324..d70c56686c5 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java @@ -513,6 +513,7 @@ default int getNowInSeconds() { * @see #NO_NOW_IN_SECONDS */ @NonNull + @CheckReturnValue @SuppressWarnings("unchecked") default SelfT setNowInSeconds(int nowInSeconds) { return (SelfT) this; From 8444c79ff843a072e5c1a1d8de5140a47051e2a0 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 16 Sep 2024 16:30:43 -0500 Subject: [PATCH 14/77] Remove "beta" support for Java17 from docs patch by Bret McGuire; reviewed by Andy Tolbert and Alexandre Dutra reference: https://github.com/apache/cassandra-java-driver/pull/1962 --- upgrade_guide/README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index c6df74ffc2a..56d55aaab36 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -19,7 +19,7 @@ under the License. ## Upgrade guide -### NEW VERSION PLACEHOLDER +### 4.18.1 #### Keystore reloading in DefaultSslEngineFactory @@ -32,12 +32,9 @@ This feature is disabled by default for compatibility. To enable, see `keystore- ### 4.17.0 -#### Beta support for Java17 +#### Support for Java17 With the completion of [JAVA-3042](https://datastax-oss.atlassian.net/browse/JAVA-3042) the driver now passes our automated test matrix for Java Driver releases. -While all features function normally when run with Java 17 tests, we do not offer full support for this -platform until we've received feedback from other users in the ecosystem. - If you discover an issue with the Java Driver running on Java 17, please let us know. We will triage and address Java 17 issues. #### Updated API for vector search From a40e7587b175cc198fb533eadabd31e94f837369 Mon Sep 17 00:00:00 2001 From: Christian Aistleitner Date: Thu, 6 Jun 2024 09:14:16 +0200 Subject: [PATCH 15/77] Fix uncaught exception during graceful channel shutdown after exceeding max orphan ids patch by Christian Aistleitner; reviewed by Andy Tolbert, and Bret McGuire for #1938 --- .../core/channel/InFlightHandler.java | 4 +- .../core/channel/InFlightHandlerTest.java | 63 ++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java index 9060f80b7cd..90b02f358cd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java @@ -199,14 +199,14 @@ private void startGracefulShutdown(ChannelHandlerContext ctx) { LOG.debug("[{}] No pending queries, completing graceful shutdown now", logPrefix); ctx.channel().close(); } else { - // remove heartbeat handler from pipeline if present. + // Remove heartbeat handler from pipeline if present. ChannelHandler heartbeatHandler = ctx.pipeline().get(ChannelFactory.HEARTBEAT_HANDLER_NAME); if (heartbeatHandler != null) { ctx.pipeline().remove(heartbeatHandler); } LOG.debug("[{}] There are pending queries, delaying graceful shutdown", logPrefix); closingGracefully = true; - closeStartedFuture.setSuccess(); + closeStartedFuture.trySuccess(); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java index 79a575d9eb6..35049e99af1 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java @@ -39,7 +39,9 @@ import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelPromise; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.Test; @@ -256,7 +258,7 @@ public void should_refuse_new_writes_during_graceful_close() { } @Test - public void should_close_gracefully_if_orphan_ids_above_max_and_pending_requests() { + public void should_close_gracefully_if_orphan_ids_above_max_and_pending_request() { // Given addToPipeline(); // Generate n orphan ids by writing and cancelling the requests: @@ -311,6 +313,65 @@ public void should_close_gracefully_if_orphan_ids_above_max_and_pending_requests assertThat(channel.closeFuture()).isSuccess(); } + @Test + public void should_close_gracefully_if_orphan_ids_above_max_and_multiple_pending_requests() { + // Given + addToPipeline(); + // Generate n orphan ids by writing and cancelling the requests. + for (int i = 0; i < MAX_ORPHAN_IDS; i++) { + when(streamIds.acquire()).thenReturn(i); + MockResponseCallback responseCallback = new MockResponseCallback(); + channel + .writeAndFlush( + new DriverChannel.RequestMessage(QUERY, false, Frame.NO_PAYLOAD, responseCallback)) + .awaitUninterruptibly(); + channel.writeAndFlush(responseCallback).awaitUninterruptibly(); + } + // Generate 3 additional requests that are pending and not cancelled. + List pendingResponseCallbacks = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + when(streamIds.acquire()).thenReturn(MAX_ORPHAN_IDS + i); + MockResponseCallback responseCallback = new MockResponseCallback(); + channel + .writeAndFlush( + new DriverChannel.RequestMessage(QUERY, false, Frame.NO_PAYLOAD, responseCallback)) + .awaitUninterruptibly(); + pendingResponseCallbacks.add(responseCallback); + } + + // When + // Generate the n+1th orphan id that makes us go above the threshold by canceling one if the + // pending requests. + channel.writeAndFlush(pendingResponseCallbacks.remove(0)).awaitUninterruptibly(); + + // Then + // Channel should be closing gracefully but there's no way to observe that from the outside + // besides writing another request and check that it's rejected. + assertThat(channel.closeFuture()).isNotDone(); + ChannelFuture otherWriteFuture = + channel.writeAndFlush( + new DriverChannel.RequestMessage( + QUERY, false, Frame.NO_PAYLOAD, new MockResponseCallback())); + assertThat(otherWriteFuture).isFailed(); + assertThat(otherWriteFuture.cause()) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Channel is closing"); + + // When + // Cancel the remaining pending requests causing the n+ith orphan ids above the threshold. + for (MockResponseCallback pendingResponseCallback : pendingResponseCallbacks) { + ChannelFuture future = channel.writeAndFlush(pendingResponseCallback).awaitUninterruptibly(); + + // Then + // The future should succeed even though the channel has started closing gracefully. + assertThat(future).isSuccess(); + } + + // Then + // The graceful shutdown completes. + assertThat(channel.closeFuture()).isSuccess(); + } + @Test public void should_close_immediately_if_orphan_ids_above_max_and_no_pending_requests() { // Given From eb57fd7d46fb8b84655e66a3ca8e9ceef77b5164 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 18 Sep 2024 08:49:37 +0000 Subject: [PATCH 16/77] Build a public CI for Apache Cassandra Java Driver patch by Siyao (Jane) He; reviewed by Mick Semb Wever for CASSANDRA-19832 --- Jenkinsfile-asf | 80 +++++++++++++++++++++++++++++ Jenkinsfile => Jenkinsfile-datastax | 0 ci/create-user.sh | 60 ++++++++++++++++++++++ ci/run-tests.sh | 10 ++++ 4 files changed, 150 insertions(+) create mode 100644 Jenkinsfile-asf rename Jenkinsfile => Jenkinsfile-datastax (100%) create mode 100644 ci/create-user.sh create mode 100755 ci/run-tests.sh diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf new file mode 100644 index 00000000000..a1be4bcd4f6 --- /dev/null +++ b/Jenkinsfile-asf @@ -0,0 +1,80 @@ +#!groovy + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +pipeline { + agent { + label 'cassandra-small' + } + + triggers { + // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) + cron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? '@weekly' : '') + } + + stages { + stage('Matrix') { + matrix { + axes { + axis { + name 'TEST_JAVA_VERSION' + values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@17' + } + axis { + name 'SERVER_VERSION' + values '3.11.17', + '4.0.13', + '5.0-beta1' + } + } + stages { + stage('Tests') { + agent { + label 'cassandra-medium' + } + steps { + script { + executeTests() + junit testResults: '**/target/surefire-reports/TEST-*.xml', allowEmptyResults: true + junit testResults: '**/target/failsafe-reports/TEST-*.xml', allowEmptyResults: true + } + } + } + } + } + } + } +} + +def executeTests() { + def testJavaMajorVersion = (TEST_JAVA_VERSION =~ /@(?:1\.)?(\d+)/)[0][1] + sh """ + container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver apache.jfrog.io/cassan-docker/apache/cassandra-java-driver-testing-ubuntu2204 'sleep 2h') + docker exec --user root \$container_id bash -c \"sudo bash /home/docker/cassandra-java-driver/ci/create-user.sh docker \$(id -u) \$(id -g) /home/docker/cassandra-java-driver\" + docker exec --user docker \$container_id './cassandra-java-driver/ci/run-tests.sh' + ( nohup docker stop \$container_id >/dev/null 2>/dev/null & ) + """ +} + +// branch pattern for cron +// should match 3.x, 4.x, 4.5.x, etc +def branchPatternCron() { + ~'((\\d+(\\.[\\dx]+)+))' +} diff --git a/Jenkinsfile b/Jenkinsfile-datastax similarity index 100% rename from Jenkinsfile rename to Jenkinsfile-datastax diff --git a/ci/create-user.sh b/ci/create-user.sh new file mode 100644 index 00000000000..fb193df9a00 --- /dev/null +++ b/ci/create-user.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +if [ "$1" == "-h" ]; then + echo "$0 [-h] " + echo " this script is used internally by other scripts in the same directory to create a user with the running host user's same uid and gid" + exit 1 +fi + +# arguments +username=$1 +uid=$2 +gid=$3 +BUILD_HOME=$4 + +################################ +# +# Main +# +################################ + +# disable git directory ownership checks +su ${username} -c "git config --global safe.directory '*'" + +if grep "^ID=" /etc/os-release | grep -q 'debian\|ubuntu' ; then + deluser docker + adduser --quiet --disabled-login --no-create-home --uid $uid --gecos ${username} ${username} + groupmod --non-unique -g $gid $username + gpasswd -a ${username} sudo >/dev/null +else + adduser --no-create-home --uid $uid ${username} +fi + +# sudo priviledges +echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} +chmod 0440 /etc/sudoers.d/${username} + +# proper permissions +chown -R ${username}:${username} /home/docker +chmod og+wx ${BUILD_HOME} \ No newline at end of file diff --git a/ci/run-tests.sh b/ci/run-tests.sh new file mode 100755 index 00000000000..02a8070e7a9 --- /dev/null +++ b/ci/run-tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash -x + +. ~/.jabba/jabba.sh +. ~/env.txt +cd $(dirname "$(readlink -f "$0")")/.. +printenv | sort +mvn -B -V install -DskipTests -Dmaven.javadoc.skip=true +jabba use ${TEST_JAVA_VERSION} +printenv | sort +mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true From 72c729b1ba95695fed467ca3734de7a39a2b3201 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Thu, 3 Oct 2024 09:32:17 +0200 Subject: [PATCH 17/77] CASSANDRA-19932: Allow to define extensions while creating table patch by Lukasz Antoniak; reviewed by Bret McGuire and Chris Lohfink --- .../schema/CreateTableWithOptions.java | 11 +++++ .../schema/RawOptionsWrapper.java | 45 +++++++++++++++++++ .../querybuilder/schema/CreateTableTest.java | 9 +++- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java index 4dd3193da15..c7bddf575fb 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java @@ -18,7 +18,11 @@ package com.datastax.oss.driver.api.querybuilder.schema; import com.datastax.oss.driver.api.querybuilder.BuildableQuery; +import com.datastax.oss.driver.internal.querybuilder.schema.RawOptionsWrapper; +import com.datastax.oss.driver.shaded.guava.common.collect.Maps; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Map; public interface CreateTableWithOptions extends BuildableQuery, RelationStructure { @@ -26,4 +30,11 @@ public interface CreateTableWithOptions /** Enables COMPACT STORAGE in the CREATE TABLE statement. */ @NonNull CreateTableWithOptions withCompactStorage(); + + /** Attaches custom metadata to CQL table definition. */ + @NonNull + @CheckReturnValue + default CreateTableWithOptions withExtensions(@NonNull Map extensions) { + return withOption("extensions", Maps.transformValues(extensions, RawOptionsWrapper::of)); + } } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java new file mode 100644 index 00000000000..64cdb50f887 --- /dev/null +++ b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.querybuilder.schema; + +import com.datastax.oss.driver.api.core.data.ByteUtils; + +/** + * Wrapper class to indicate that the contained String value should be understood to represent a CQL + * literal that can be included directly in a CQL statement (i.e. without escaping). + */ +public class RawOptionsWrapper { + private final String val; + + private RawOptionsWrapper(String val) { + this.val = val; + } + + public static RawOptionsWrapper of(String val) { + return new RawOptionsWrapper(val); + } + + public static RawOptionsWrapper of(byte[] val) { + return new RawOptionsWrapper(ByteUtils.toHexString(val)); + } + + @Override + public String toString() { + return this.val; + } +} diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java index d32c66f629b..7a5542c51f0 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.querybuilder.schema.compaction.TimeWindowCompactionStrategy.CompactionWindowUnit; import com.datastax.oss.driver.api.querybuilder.schema.compaction.TimeWindowCompactionStrategy.TimestampResolution; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.nio.charset.StandardCharsets; import org.junit.Test; public class CreateTableTest { @@ -169,6 +170,12 @@ public void should_generate_create_table_with_options() { .withComment("Hello world") .withDcLocalReadRepairChance(0.54) .withDefaultTimeToLiveSeconds(86400) + .withExtensions( + ImmutableMap.of( + "key1", + "apache".getBytes(StandardCharsets.UTF_8), + "key2", + "cassandra".getBytes(StandardCharsets.UTF_8))) .withGcGraceSeconds(864000) .withMemtableFlushPeriodInMs(10000) .withMinIndexInterval(1024) @@ -176,7 +183,7 @@ public void should_generate_create_table_with_options() { .withReadRepairChance(0.55) .withSpeculativeRetry("99percentile")) .hasCql( - "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH bloom_filter_fp_chance=0.42 AND cdc=false AND comment='Hello world' AND dclocal_read_repair_chance=0.54 AND default_time_to_live=86400 AND gc_grace_seconds=864000 AND memtable_flush_period_in_ms=10000 AND min_index_interval=1024 AND max_index_interval=4096 AND read_repair_chance=0.55 AND speculative_retry='99percentile'"); + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH bloom_filter_fp_chance=0.42 AND cdc=false AND comment='Hello world' AND dclocal_read_repair_chance=0.54 AND default_time_to_live=86400 AND extensions={'key1':0x617061636865,'key2':0x63617373616e647261} AND gc_grace_seconds=864000 AND memtable_flush_period_in_ms=10000 AND min_index_interval=1024 AND max_index_interval=4096 AND read_repair_chance=0.55 AND speculative_retry='99percentile'"); } @Test From 8ebcd9f85afb548f38e953fb1190d9ff04d8df5a Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Tue, 15 Oct 2024 19:24:00 -0400 Subject: [PATCH 18/77] Fix DefaultSslEngineFactory missing null check on close patch by Abe Ratnofsky; reviewed by Andy Tolbert and Chris Lohfink for CASSANDRA-20001 --- .../oss/driver/internal/core/ssl/DefaultSslEngineFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java index bb95dc738c7..475ec38d578 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java @@ -164,6 +164,6 @@ private ReloadingKeyManagerFactory buildReloadingKeyManagerFactory(DriverExecuti @Override public void close() throws Exception { - kmf.close(); + if (kmf != null) kmf.close(); } } From f98e3433b91b49e0facfbce8e94e01e304714968 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 2 Oct 2024 18:04:19 -0500 Subject: [PATCH 19/77] Query builder support for NOT CQL syntax patch by Bret McGuire; reviewed by Bret McGuire and Andy Tolbert for CASSANDRA-19930 --- .../relation/ColumnRelationBuilder.java | 24 +++++++ .../relation/InRelationBuilder.java | 32 +++++++++ .../querybuilder/relation/RelationTest.java | 69 ++++++++++++++++++- 3 files changed, 124 insertions(+), 1 deletion(-) diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java index 613e72291b7..247d61eaed5 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java @@ -46,4 +46,28 @@ default ResultT contains(@NonNull Term term) { default ResultT containsKey(@NonNull Term term) { return build(" CONTAINS KEY ", term); } + + /** + * Builds a NOT CONTAINS relation for the column. + * + *

Note that NOT CONTAINS support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notContains(@NonNull Term term) { + return build(" NOT CONTAINS ", term); + } + + /** + * Builds a NOT CONTAINS KEY relation for the column. + * + *

Note that NOT CONTAINS KEY support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notContainsKey(@NonNull Term term) { + return build(" NOT CONTAINS KEY ", term); + } } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java index d3fc8dce91d..afaa19ff724 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java @@ -50,6 +50,38 @@ default ResultT in(@NonNull Term... alternatives) { return in(Arrays.asList(alternatives)); } + /** + * Builds a NOT IN relation where the whole set of possible values is a bound variable, as in + * {@code NOT IN ?}. + * + *

Note that NOT IN support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notIn(@NonNull BindMarker bindMarker) { + return build(" NOT IN ", bindMarker); + } + + /** + * Builds an IN relation where the arguments are the possible values, as in {@code IN (term1, + * term2...)}. + * + *

Note that NOT IN support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notIn(@NonNull Iterable alternatives) { + return build(" NOT IN ", QueryBuilder.tuple(alternatives)); + } + + /** Var-arg equivalent of {@link #notIn(Iterable)} . */ + @NonNull + default ResultT notIn(@NonNull Term... alternatives) { + return notIn(Arrays.asList(alternatives)); + } + @NonNull ResultT build(@NonNull String operator, @Nullable Term rightOperand); } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java index 515a336f5f4..ec121eaa050 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java @@ -19,10 +19,12 @@ import static com.datastax.oss.driver.api.querybuilder.Assertions.assertThat; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.raw; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.tuple; +import org.assertj.core.util.Lists; import org.junit.Test; public class RelationTest { @@ -42,13 +44,78 @@ public void should_generate_is_not_null_relation() { } @Test - public void should_generate_in_relation() { + public void should_generate_contains_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").contains(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k CONTAINS 1"); + } + + @Test + public void should_generate_contains_key_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").containsKey(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k CONTAINS KEY 1"); + } + + @Test + public void should_generate_not_contains_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notContains(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k NOT CONTAINS 1"); + } + + @Test + public void should_generate_not_contains_key_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notContainsKey(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k NOT CONTAINS KEY 1"); + } + + @Test + public void should_generate_in_relation_bind_markers() { assertThat(selectFrom("foo").all().where(Relation.column("k").in(bindMarker()))) .hasCql("SELECT * FROM foo WHERE k IN ?"); assertThat(selectFrom("foo").all().where(Relation.column("k").in(bindMarker(), bindMarker()))) .hasCql("SELECT * FROM foo WHERE k IN (?,?)"); } + @Test + public void should_generate_in_relation_terms() { + assertThat( + selectFrom("foo") + .all() + .where( + Relation.column("k") + .in(Lists.newArrayList(literal(1), literal(2), literal(3))))) + .hasCql("SELECT * FROM foo WHERE k IN (1,2,3)"); + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").in(literal(1), literal(2), literal(3)))) + .hasCql("SELECT * FROM foo WHERE k IN (1,2,3)"); + } + + @Test + public void should_generate_not_in_relation_bind_markers() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notIn(bindMarker()))) + .hasCql("SELECT * FROM foo WHERE k NOT IN ?"); + assertThat( + selectFrom("foo").all().where(Relation.column("k").notIn(bindMarker(), bindMarker()))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (?,?)"); + } + + @Test + public void should_generate_not_in_relation_terms() { + assertThat( + selectFrom("foo") + .all() + .where( + Relation.column("k") + .notIn(Lists.newArrayList(literal(1), literal(2), literal(3))))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (1,2,3)"); + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").notIn(literal(1), literal(2), literal(3)))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (1,2,3)"); + } + @Test public void should_generate_token_relation() { assertThat(selectFrom("foo").all().where(Relation.token("k1", "k2").isEqualTo(bindMarker("t")))) From dfe11a8b671d76be6c4e90981a736325b0e4719b Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Sep 2024 06:43:15 -0400 Subject: [PATCH 20/77] Fix CustomCcmRule to drop `CURRENT` flag no matter what If super.after() throws an Exception `CURRENT` flag is never dropped which leads next tests to fail with IllegalStateException("Attempting to use a Ccm rule while another is in use. This is disallowed") Patch by Dmitry Kropachev; reviewed by Andy Tolbert and Bret McGuire for JAVA-3117 --- .../oss/driver/api/testinfra/ccm/CustomCcmRule.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java index cf150b12f55..5ea1bf7ed3c 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java @@ -53,6 +53,7 @@ protected void before() { after(); } catch (Exception e1) { LOG.warn("Error cleaning up CustomCcmRule before() failure", e1); + e.addSuppressed(e1); } throw e; } @@ -64,8 +65,11 @@ protected void before() { @Override protected void after() { - super.after(); - CURRENT.compareAndSet(this, null); + try { + super.after(); + } finally { + CURRENT.compareAndSet(this, null); + } } public CcmBridge getCcmBridge() { From 787783770b0a624f4e58d35aeff16eff8bcddc02 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 30 Oct 2024 11:34:11 -0700 Subject: [PATCH 21/77] JAVA-3051: Memory leak patch by Jane He; reviewed by Alexandre Dutra and Bret McGuire for JAVA-3051 --- .../core/control/ControlConnection.java | 21 ++-- .../DefaultLoadBalancingPolicy.java | 96 ++++++++++++------- .../metadata/LoadBalancingPolicyWrapper.java | 12 ++- .../core/metrics/AbstractMetricUpdater.java | 5 +- .../internal/core/session/DefaultSession.java | 14 ++- .../util/concurrent/ReplayingEventFilter.java | 1 + ...faultLoadBalancingPolicyQueryPlanTest.java | 9 +- ...LoadBalancingPolicyRequestTrackerTest.java | 52 +++++----- 8 files changed, 126 insertions(+), 84 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 5ee9c6e7810..5c29a9b704b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -253,8 +253,8 @@ private class SingleThreaded { private final Reconnection reconnection; private DriverChannelOptions channelOptions; // The last events received for each node - private final Map lastDistanceEvents = new WeakHashMap<>(); - private final Map lastStateEvents = new WeakHashMap<>(); + private final Map lastNodeDistance = new WeakHashMap<>(); + private final Map lastNodeState = new WeakHashMap<>(); private SingleThreaded(InternalDriverContext context) { this.context = context; @@ -366,8 +366,8 @@ private void connect( .whenCompleteAsync( (channel, error) -> { try { - DistanceEvent lastDistanceEvent = lastDistanceEvents.get(node); - NodeStateEvent lastStateEvent = lastStateEvents.get(node); + NodeDistance lastDistance = lastNodeDistance.get(node); + NodeState lastState = lastNodeState.get(node); if (error != null) { if (closeWasCalled || initFuture.isCancelled()) { onSuccess.run(); // abort, we don't really care about the result @@ -406,8 +406,7 @@ private void connect( channel); channel.forceClose(); onSuccess.run(); - } else if (lastDistanceEvent != null - && lastDistanceEvent.distance == NodeDistance.IGNORED) { + } else if (lastDistance == NodeDistance.IGNORED) { LOG.debug( "[{}] New channel opened ({}) but node became ignored, " + "closing and trying next node", @@ -415,9 +414,9 @@ private void connect( channel); channel.forceClose(); connect(nodes, errors, onSuccess, onFailure); - } else if (lastStateEvent != null - && (lastStateEvent.newState == null /*(removed)*/ - || lastStateEvent.newState == NodeState.FORCED_DOWN)) { + } else if (lastNodeState.containsKey(node) + && (lastState == null /*(removed)*/ + || lastState == NodeState.FORCED_DOWN)) { LOG.debug( "[{}] New channel opened ({}) but node was removed or forced down, " + "closing and trying next node", @@ -534,7 +533,7 @@ private void reconnectNow() { private void onDistanceEvent(DistanceEvent event) { assert adminExecutor.inEventLoop(); - this.lastDistanceEvents.put(event.node, event); + this.lastNodeDistance.put(event.node, event.distance); if (event.distance == NodeDistance.IGNORED && channel != null && !channel.closeFuture().isDone() @@ -549,7 +548,7 @@ private void onDistanceEvent(DistanceEvent event) { private void onStateEvent(NodeStateEvent event) { assert adminExecutor.inEventLoop(); - this.lastStateEvents.put(event.node, event); + this.lastNodeState.put(event.node, event.newState); if ((event.newState == null /*(removed)*/ || event.newState == NodeState.FORCED_DOWN) && channel != null && !channel.closeFuture().isDone() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java index 47edcdfe53e..0f03cbb3643 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java @@ -33,15 +33,19 @@ import com.datastax.oss.driver.internal.core.util.ArrayUtils; import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import com.datastax.oss.driver.shaded.guava.common.collect.MapMaker; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.BitSet; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Queue; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLongArray; import net.jcip.annotations.ThreadSafe; @@ -96,7 +100,7 @@ public class DefaultLoadBalancingPolicy extends BasicLoadBalancingPolicy impleme private static final int MAX_IN_FLIGHT_THRESHOLD = 10; private static final long RESPONSE_COUNT_RESET_INTERVAL_NANOS = MILLISECONDS.toNanos(200); - protected final Map responseTimes = new ConcurrentHashMap<>(); + protected final ConcurrentMap responseTimes; protected final Map upTimes = new ConcurrentHashMap<>(); private final boolean avoidSlowReplicas; @@ -104,6 +108,7 @@ public DefaultLoadBalancingPolicy(@NonNull DriverContext context, @NonNull Strin super(context, profileName); this.avoidSlowReplicas = profile.getBoolean(DefaultDriverOption.LOAD_BALANCING_POLICY_SLOW_AVOIDANCE, true); + this.responseTimes = new MapMaker().weakKeys().makeMap(); } @NonNull @@ -274,40 +279,19 @@ protected boolean isBusy(@NonNull Node node, @NonNull Session session) { } protected boolean isResponseRateInsufficient(@NonNull Node node, long now) { - // response rate is considered insufficient when less than 2 responses were obtained in - // the past interval delimited by RESPONSE_COUNT_RESET_INTERVAL_NANOS. - if (responseTimes.containsKey(node)) { - AtomicLongArray array = responseTimes.get(node); - if (array.length() == 2) { - long threshold = now - RESPONSE_COUNT_RESET_INTERVAL_NANOS; - long leastRecent = array.get(0); - return leastRecent - threshold < 0; - } - } - return true; + NodeResponseRateSample sample = responseTimes.get(node); + return !(sample == null || sample.hasSufficientResponses(now)); } + /** + * Synchronously updates the response times for the given node. It is synchronous because the + * {@link #DefaultLoadBalancingPolicy(com.datastax.oss.driver.api.core.context.DriverContext, + * java.lang.String) CacheLoader.load} assigned is synchronous. + * + * @param node The node to update. + */ protected void updateResponseTimes(@NonNull Node node) { - responseTimes.compute( - node, - (n, array) -> { - // The array stores at most two timestamps, since we don't need more; - // the first one is always the least recent one, and hence the one to inspect. - long now = nanoTime(); - if (array == null) { - array = new AtomicLongArray(1); - array.set(0, now); - } else if (array.length() == 1) { - long previous = array.get(0); - array = new AtomicLongArray(2); - array.set(0, previous); - array.set(1, now); - } else { - array.set(0, array.get(1)); - array.set(1, now); - } - return array; - }); + this.responseTimes.compute(node, (k, v) -> v == null ? new NodeResponseRateSample() : v.next()); } protected int getInFlight(@NonNull Node node, @NonNull Session session) { @@ -318,4 +302,52 @@ protected int getInFlight(@NonNull Node node, @NonNull Session session) { // processing them). return (pool == null) ? 0 : pool.getInFlight(); } + + protected class NodeResponseRateSample { + + @VisibleForTesting protected final long oldest; + @VisibleForTesting protected final OptionalLong newest; + + private NodeResponseRateSample() { + long now = nanoTime(); + this.oldest = now; + this.newest = OptionalLong.empty(); + } + + private NodeResponseRateSample(long oldestSample) { + this(oldestSample, nanoTime()); + } + + private NodeResponseRateSample(long oldestSample, long newestSample) { + this.oldest = oldestSample; + this.newest = OptionalLong.of(newestSample); + } + + @VisibleForTesting + protected NodeResponseRateSample(AtomicLongArray times) { + assert times.length() >= 1; + this.oldest = times.get(0); + this.newest = (times.length() > 1) ? OptionalLong.of(times.get(1)) : OptionalLong.empty(); + } + + // Our newest sample becomes the oldest in the next generation + private NodeResponseRateSample next() { + return new NodeResponseRateSample(this.getNewestValidSample(), nanoTime()); + } + + // If we have a pair of values return the newest, otherwise we have just one value... so just + // return it + private long getNewestValidSample() { + return this.newest.orElse(this.oldest); + } + + // response rate is considered insufficient when less than 2 responses were obtained in + // the past interval delimited by RESPONSE_COUNT_RESET_INTERVAL_NANOS. + private boolean hasSufficientResponses(long now) { + // If we only have one sample it's an automatic failure + if (!this.newest.isPresent()) return true; + long threshold = now - RESPONSE_COUNT_RESET_INTERVAL_NANOS; + return this.oldest - threshold >= 0; + } + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index 20d045d4e72..5c8473a3b67 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -105,7 +106,7 @@ public LoadBalancingPolicyWrapper( // Just an alias to make the rest of the code more readable this.policies = reporters.keySet(); - this.distances = new HashMap<>(); + this.distances = new WeakHashMap<>(); this.logPrefix = context.getSessionName(); context.getEventBus().register(NodeStateEvent.class, this::onNodeStateEvent); @@ -172,6 +173,7 @@ private void onNodeStateEvent(NodeStateEvent event) { // once it has gone through the filter private void processNodeStateEvent(NodeStateEvent event) { + DefaultNode node = event.node; switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: @@ -181,13 +183,13 @@ private void processNodeStateEvent(NodeStateEvent event) { case RUNNING: for (LoadBalancingPolicy policy : policies) { if (event.newState == NodeState.UP) { - policy.onUp(event.node); + policy.onUp(node); } else if (event.newState == NodeState.DOWN || event.newState == NodeState.FORCED_DOWN) { - policy.onDown(event.node); + policy.onDown(node); } else if (event.newState == NodeState.UNKNOWN) { - policy.onAdd(event.node); + policy.onAdd(node); } else if (event.newState == null) { - policy.onRemove(event.node); + policy.onRemove(node); } else { LOG.warn("[{}] Unsupported event: {}", logPrefix, event); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java index 5e2392a2e7f..3d7dc50a7c0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java @@ -173,9 +173,8 @@ protected Timeout newTimeout() { .getTimer() .newTimeout( t -> { - if (t.isExpired()) { - clearMetrics(); - } + clearMetrics(); + cancelMetricsExpirationTimeout(); }, expireAfter.toNanos(), TimeUnit.NANOSECONDS); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java index 6f063ae9a50..b795c30fce7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java @@ -527,14 +527,18 @@ private void notifyListeners() { private void onNodeStateChanged(NodeStateEvent event) { assert adminExecutor.inEventLoop(); - if (event.newState == null) { - context.getNodeStateListener().onRemove(event.node); + DefaultNode node = event.node; + if (node == null) { + LOG.debug( + "[{}] Node for this event was removed, ignoring state change: {}", logPrefix, event); + } else if (event.newState == null) { + context.getNodeStateListener().onRemove(node); } else if (event.oldState == null && event.newState == NodeState.UNKNOWN) { - context.getNodeStateListener().onAdd(event.node); + context.getNodeStateListener().onAdd(node); } else if (event.newState == NodeState.UP) { - context.getNodeStateListener().onUp(event.node); + context.getNodeStateListener().onUp(node); } else if (event.newState == NodeState.DOWN || event.newState == NodeState.FORCED_DOWN) { - context.getNodeStateListener().onDown(event.node); + context.getNodeStateListener().onDown(node); } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java index 12679db7ff0..27ca1b6ff42 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java @@ -82,6 +82,7 @@ public void markReady() { consumer.accept(event); } } finally { + recordedEvents.clear(); stateLock.writeLock().unlock(); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java index 6098653bc2e..fff86a1b750 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java @@ -203,7 +203,10 @@ public void should_prioritize_and_shuffle_3_or_more_replicas_when_first_unhealth given(pool3.getInFlight()).willReturn(0); given(pool5.getInFlight()).willReturn(0); - dsePolicy.responseTimes.put(node1, new AtomicLongArray(new long[] {T0, T0})); // unhealthy + dsePolicy.responseTimes.put( + node1, + dsePolicy + .new NodeResponseRateSample(new AtomicLongArray(new long[] {T0, T0}))); // unhealthy // When Queue plan1 = dsePolicy.newQueryPlan(request, session); @@ -232,7 +235,9 @@ public void should_prioritize_and_shuffle_3_or_more_replicas_when_first_unhealth given(pool3.getInFlight()).willReturn(0); given(pool5.getInFlight()).willReturn(0); - dsePolicy.responseTimes.put(node1, new AtomicLongArray(new long[] {T1, T1})); // healthy + dsePolicy.responseTimes.put( + node1, + dsePolicy.new NodeResponseRateSample(new AtomicLongArray(new long[] {T1, T1}))); // healthy // When Queue plan1 = dsePolicy.newQueryPlan(request, session); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java index bcc6439a2a5..757af43ef67 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java @@ -69,11 +69,11 @@ public void should_record_first_response_time_on_node_success() { // Then assertThat(policy.responseTimes) - .hasEntrySatisfying(node1, value -> assertThat(value.get(0)).isEqualTo(123L)) + .hasEntrySatisfying(node1, value -> assertThat(value.oldest).isEqualTo(123L)) .doesNotContainKeys(node2, node3); - assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -91,13 +91,13 @@ public void should_record_second_response_time_on_node_success() { node1, value -> { // oldest value first - assertThat(value.get(0)).isEqualTo(123); - assertThat(value.get(1)).isEqualTo(456); + assertThat(value.oldest).isEqualTo(123); + assertThat(value.newest.getAsLong()).isEqualTo(456); }) .doesNotContainKeys(node2, node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -116,14 +116,14 @@ public void should_record_further_response_times_on_node_success() { node1, value -> { // values should rotate left (bubble up) - assertThat(value.get(0)).isEqualTo(456); - assertThat(value.get(1)).isEqualTo(789); + assertThat(value.oldest).isEqualTo(456); + assertThat(value.newest.getAsLong()).isEqualTo(789); }) - .hasEntrySatisfying(node2, value -> assertThat(value.get(0)).isEqualTo(789)) + .hasEntrySatisfying(node2, value -> assertThat(value.oldest).isEqualTo(789)) .doesNotContainKey(node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -137,11 +137,11 @@ public void should_record_first_response_time_on_node_error() { // Then assertThat(policy.responseTimes) - .hasEntrySatisfying(node1, value -> assertThat(value.get(0)).isEqualTo(123L)) + .hasEntrySatisfying(node1, value -> assertThat(value.oldest).isEqualTo(123L)) .doesNotContainKeys(node2, node3); - assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -160,13 +160,13 @@ public void should_record_second_response_time_on_node_error() { node1, value -> { // oldest value first - assertThat(value.get(0)).isEqualTo(123); - assertThat(value.get(1)).isEqualTo(456); + assertThat(value.oldest).isEqualTo(123); + assertThat(value.newest.getAsLong()).isEqualTo(456); }) .doesNotContainKeys(node2, node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -186,13 +186,13 @@ public void should_record_further_response_times_on_node_error() { node1, value -> { // values should rotate left (bubble up) - assertThat(value.get(0)).isEqualTo(456); - assertThat(value.get(1)).isEqualTo(789); + assertThat(value.oldest).isEqualTo(456); + assertThat(value.newest.getAsLong()).isEqualTo(789); }) - .hasEntrySatisfying(node2, value -> assertThat(value.get(0)).isEqualTo(789)) + .hasEntrySatisfying(node2, value -> assertThat(value.oldest).isEqualTo(789)) .doesNotContainKey(node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } } From a4175f33e43344fd1b092aae332ed5979a1bd831 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 28 Oct 2024 14:44:22 -0700 Subject: [PATCH 22/77] Automate latest Cassandra versions when running CI patch by Siyao (Jane) He; reviewed by Mick Semb Wever for CASSJAVA-25 --- Jenkinsfile-asf | 7 ++++--- ci/run-tests.sh | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf index a1be4bcd4f6..0217d0455d6 100644 --- a/Jenkinsfile-asf +++ b/Jenkinsfile-asf @@ -39,9 +39,10 @@ pipeline { } axis { name 'SERVER_VERSION' - values '3.11.17', - '4.0.13', - '5.0-beta1' + values '3.11', + '4.0', + '4.1', + '5.0' } } stages { diff --git a/ci/run-tests.sh b/ci/run-tests.sh index 02a8070e7a9..5268bdd7113 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -6,5 +6,7 @@ cd $(dirname "$(readlink -f "$0")")/.. printenv | sort mvn -B -V install -DskipTests -Dmaven.javadoc.skip=true jabba use ${TEST_JAVA_VERSION} +# Find out the latest patch version of Cassandra +PATCH_SERVER_VERSION=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP '(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -rV | uniq -w 3 | grep $SERVER_VERSION) printenv | sort -mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true +mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${PATCH_SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true From 01c6151cd4a3c1dbbd4d1251fb453305385668e1 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Sun, 8 Sep 2024 15:06:53 +0200 Subject: [PATCH 23/77] Refactor integration tests to support multiple C* distributions. Test with DataStax HCD 1.0.0 patch by Lukasz Antoniak; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1958 --- Jenkinsfile-datastax | 34 +++- .../datastax/oss/driver/api/core/Version.java | 1 + .../cql/continuous/ContinuousPagingIT.java | 8 +- .../remote/GraphTraversalRemoteITBase.java | 5 +- .../graph/statement/GraphTraversalITBase.java | 5 +- .../schema/DseAggregateMetadataIT.java | 6 +- .../schema/DseFunctionMetadataIT.java | 6 +- .../core/compression/DirectCompressionIT.java | 6 +- .../core/compression/HeapCompressionIT.java | 3 +- .../oss/driver/core/cql/QueryTraceIT.java | 3 +- .../oss/driver/core/metadata/DescribeIT.java | 28 +-- .../driver/core/metadata/NodeMetadataIT.java | 7 +- .../driver/core/metadata/SchemaChangesIT.java | 5 +- .../oss/driver/core/metadata/SchemaIT.java | 8 +- .../oss/driver/mapper/DeleteReactiveIT.java | 5 +- .../oss/driver/mapper/InventoryITBase.java | 11 +- .../src/test/resources/DescribeIT/hcd/1.0.cql | 186 ++++++++++++++++++ osgi-tests/README.md | 4 +- .../osgi/support/CcmStagedReactor.java | 8 +- test-infra/revapi.json | 21 ++ .../driver/api/testinfra/ccm/BaseCcmRule.java | 24 ++- .../driver/api/testinfra/ccm/CcmBridge.java | 83 ++++---- .../DefaultCcmBridgeBuilderCustomizer.java | 8 +- .../ccm/DistributionCassandraVersions.java | 57 ++++++ .../requirement/BackendRequirementRule.java | 2 +- .../testinfra/requirement/BackendType.java | 13 +- .../api/testinfra/session/SessionRule.java | 13 +- 27 files changed, 439 insertions(+), 121 deletions(-) create mode 100644 integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql create mode 100644 test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index 4cc20d79604..af3faafee20 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -61,7 +61,7 @@ def initializeEnvironment() { . ${JABBA_SHELL} jabba which 1.8''', returnStdout: true).trim() - sh label: 'Download Apache CassandraⓇ or DataStax Enterprise',script: '''#!/bin/bash -le + sh label: 'Download Apache CassandraⓇ, DataStax Enterprise or DataStax HCD ',script: '''#!/bin/bash -le . ${JABBA_SHELL} jabba use 1.8 . ${CCM_ENVIRONMENT_SHELL} ${SERVER_VERSION} @@ -75,13 +75,26 @@ CCM_CASSANDRA_VERSION=${DSE_FIXED_VERSION} # maintain for backwards compatibilit CCM_VERSION=${DSE_FIXED_VERSION} CCM_SERVER_TYPE=dse DSE_VERSION=${DSE_FIXED_VERSION} -CCM_IS_DSE=true CCM_BRANCH=${DSE_FIXED_VERSION} DSE_BRANCH=${DSE_FIXED_VERSION} ENVIRONMENT_EOF ''' } + if (env.SERVER_VERSION.split('-')[0] == 'hcd') { + env.HCD_FIXED_VERSION = env.SERVER_VERSION.split('-')[1] + sh label: 'Update environment for DataStax HCD', script: '''#!/bin/bash -le + cat >> ${HOME}/environment.txt << ENVIRONMENT_EOF +CCM_CASSANDRA_VERSION=${HCD_FIXED_VERSION} # maintain for backwards compatibility +CCM_VERSION=${HCD_FIXED_VERSION} +CCM_SERVER_TYPE=hcd +HCD_VERSION=${HCD_FIXED_VERSION} +CCM_BRANCH=${HCD_FIXED_VERSION} +HCD_BRANCH=${HCD_FIXED_VERSION} +ENVIRONMENT_EOF + ''' + } + sh label: 'Display Java and environment information',script: '''#!/bin/bash -le # Load CCM environment variables set -o allexport @@ -144,7 +157,7 @@ def executeTests() { -Dmaven.test.failure.ignore=true \ -Dmaven.javadoc.skip=${SKIP_JAVADOCS} \ -Dccm.version=${CCM_CASSANDRA_VERSION} \ - -Dccm.dse=${CCM_IS_DSE} \ + -Dccm.distribution=${CCM_SERVER_TYPE:cassandra} \ -Dproxy.path=${HOME}/proxy \ ${SERIAL_ITS_ARGUMENT} \ ${ISOLATED_ITS_ARGUMENT} \ @@ -269,6 +282,7 @@ pipeline { 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0', // Current DataStax HCD 'ALL'], description: '''Apache Cassandra® and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS builds @@ -330,6 +344,10 @@ pipeline { + + + +
dse-6.9.0 DataStax Enterprise v6.9.x
hcd-1.0.0DataStax HCD v1.0.x
''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_JABBA_VERSION', @@ -421,9 +439,9 @@ pipeline { H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -460,7 +478,8 @@ pipeline { values '3.11', // Latest stable Apache CassandraⓇ '4.1', // Development Apache CassandraⓇ 'dse-6.8.30', // Current DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // Current DataStax HCD } axis { name 'JABBA_VERSION' @@ -578,7 +597,8 @@ pipeline { 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // Current DataStax HCD } } when { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java index 4de006da268..52751e02984 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java @@ -48,6 +48,7 @@ public class Version implements Comparable, Serializable { private static final Pattern pattern = Pattern.compile(VERSION_REGEXP); + @NonNull public static final Version V1_0_0 = Objects.requireNonNull(parse("1.0.0")); @NonNull public static final Version V2_1_0 = Objects.requireNonNull(parse("2.1.0")); @NonNull public static final Version V2_2_0 = Objects.requireNonNull(parse("2.2.0")); @NonNull public static final Version V3_0_0 = Objects.requireNonNull(parse("3.0.0")); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java index 24ee5c0373d..45cc84f0719 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java @@ -46,7 +46,6 @@ import java.time.Duration; import java.util.Collections; import java.util.Iterator; -import java.util.Objects; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -281,11 +280,8 @@ public void prepared_statement_paging_should_be_resilient_to_schema_change() { // dropped. Row row = it.next(); assertThat(row.getString("k")).isNotNull(); - if (ccmRule - .getDseVersion() - .orElseThrow(IllegalStateException::new) - .compareTo(Objects.requireNonNull(Version.parse("6.0.0"))) - >= 0) { + if (ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("6.0.0")) >= 0)) { // DSE 6 only, v should be null here since dropped. // Not reliable for 5.1 since we may have gotten page queued before schema changed. assertThat(row.isNull("v")).isTrue(); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java index 69949951378..3db8a7d1a12 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java @@ -30,6 +30,7 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; @@ -643,9 +644,9 @@ public void should_allow_use_of_dsl_graph_binary() { */ @Test public void should_return_correct_results_when_bulked() { - Optional dseVersion = ccmRule().getCcmBridge().getDseVersion(); Assumptions.assumeThat( - dseVersion.isPresent() && dseVersion.get().compareTo(Version.parse("5.1.2")) > 0) + CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("5.1.2")) > 0)) .isTrue(); List results = graphTraversalSource().E().label().barrier().toList(); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java index 98d9ccf1b80..5bcb01bc165 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java @@ -36,7 +36,9 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException; import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.List; @@ -598,7 +600,8 @@ public void should_allow_use_of_dsl_graph_binary() throws Exception { @Test public void should_return_correct_results_when_bulked() { Assumptions.assumeThat( - ccmRule().getCcmBridge().getDseVersion().get().compareTo(Version.parse("5.1.2")) > 0) + CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("5.1.2")) > 0)) .isTrue(); GraphResultSet rs = diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java index b0e989e86a3..4c899fa5e63 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java @@ -106,9 +106,9 @@ public void should_parse_aggregate_with_deterministic() { } private static boolean isDse6OrHigher() { - assumeThat(CCM_RULE.getDseVersion()) + assumeThat(CCM_RULE.isDistributionOf(BackendType.DSE)) .describedAs("DSE required for DseFunctionMetadata tests") - .isPresent(); - return CCM_RULE.getDseVersion().get().compareTo(DSE_6_0_0) >= 0; + .isTrue(); + return CCM_RULE.getDistributionVersion().compareTo(DSE_6_0_0) >= 0; } } diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java index 53e2d1be8f8..53559a66b1b 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java @@ -233,9 +233,9 @@ public void should_parse_function_with_deterministic_and_monotonic_on() { } private static boolean isDse6OrHigher() { - assumeThat(CCM_RULE.getDseVersion()) + assumeThat(CCM_RULE.isDistributionOf(BackendType.DSE)) .describedAs("DSE required for DseFunctionMetadata tests") - .isPresent(); - return CCM_RULE.getDseVersion().get().compareTo(DSE_6_0_0) >= 0; + .isTrue(); + return CCM_RULE.getDistributionVersion().compareTo(DSE_6_0_0) >= 0; } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java index 51f71f85b5c..3dad08f4de6 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -75,8 +76,9 @@ public static void setup() { public void should_execute_queries_with_snappy_compression() throws Exception { Assume.assumeTrue( "Snappy is not supported in OSS C* 4.0+ with protocol v5", - CCM_RULE.getDseVersion().isPresent() - || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0); + !CCM_RULE.isDistributionOf(BackendType.HCD) + && (CCM_RULE.isDistributionOf(BackendType.DSE) + || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0)); createAndCheckCluster("snappy"); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java index 466a9d87ac3..a14c3b29b21 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.IsolatedTests; @@ -79,7 +80,7 @@ public static void setup() { public void should_execute_queries_with_snappy_compression() throws Exception { Assume.assumeTrue( "Snappy is not supported in OSS C* 4.0+ with protocol v5", - CCM_RULE.getDseVersion().isPresent() + CCM_RULE.isDistributionOf(BackendType.DSE) || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0); createAndCheckCluster("snappy"); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java index f4ac85d6629..37a600efbc4 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import java.net.InetAddress; @@ -82,7 +83,7 @@ public void should_fetch_trace_when_tracing_enabled() { InetAddress nodeAddress = ((InetSocketAddress) contactPoint.resolve()).getAddress(); boolean expectPorts = CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) >= 0 - && !CCM_RULE.getDseVersion().isPresent(); + && !CCM_RULE.isDistributionOf(BackendType.DSE); QueryTrace queryTrace = executionInfo.getQueryTrace(); assertThat(queryTrace.getTracingId()).isEqualTo(executionInfo.getTracingId()); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java index 9fbf5e355eb..4d6c2a7a3b1 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -37,12 +38,14 @@ import com.datastax.oss.driver.internal.core.metadata.schema.DefaultTableMetadata; import com.datastax.oss.driver.shaded.guava.common.base.Charsets; import com.datastax.oss.driver.shaded.guava.common.base.Splitter; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.google.common.io.Files; import java.io.File; import java.io.IOException; import java.net.URL; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import org.junit.BeforeClass; @@ -79,17 +82,23 @@ public class DescribeIT { Splitter.on(Pattern.compile(";\n")).omitEmptyStrings(); private static Version serverVersion; - private static boolean isDse; + + private static final Map scriptFileForBackend = + ImmutableMap.builder() + .put(BackendType.CASSANDRA, "DescribeIT/oss") + .put(BackendType.DSE, "DescribeIT/dse") + .put(BackendType.HCD, "DescribeIT/hcd") + .build(); private static File scriptFile; private static String scriptContents; @BeforeClass public static void setup() { - Optional dseVersion = CCM_RULE.getDseVersion(); - isDse = dseVersion.isPresent(); serverVersion = - isDse ? dseVersion.get().nextStable() : CCM_RULE.getCassandraVersion().nextStable(); + CCM_RULE.isDistributionOf(BackendType.CASSANDRA) + ? CCM_RULE.getCassandraVersion().nextStable() + : CCM_RULE.getDistributionVersion().nextStable(); scriptFile = getScriptFile(); assertThat(scriptFile).exists(); @@ -114,12 +123,12 @@ public void describe_output_should_match_creation_script() throws Exception { "Describe output doesn't match create statements, " + "maybe you need to add a new script in integration-tests/src/test/resources. " + "Server version = %s %s, used script = %s", - isDse ? "DSE" : "Cassandra", serverVersion, scriptFile) + CCM_RULE.getDistribution(), serverVersion, scriptFile) .isEqualTo(scriptContents); } private boolean atLeastVersion(Version dseVersion, Version ossVersion) { - Version comparison = isDse ? dseVersion : ossVersion; + Version comparison = CCM_RULE.isDistributionOf(BackendType.DSE) ? dseVersion : ossVersion; return serverVersion.compareTo(comparison) >= 0; } @@ -138,11 +147,9 @@ public void keyspace_metadata_should_be_serializable() throws Exception { assertThat(ks.getUserDefinedTypes()).isNotEmpty(); assertThat(ks.getTables()).isNotEmpty(); if (atLeastVersion(Version.V5_0_0, Version.V3_0_0)) { - assertThat(ks.getViews()).isNotEmpty(); } if (atLeastVersion(Version.V5_0_0, Version.V2_2_0)) { - assertThat(ks.getFunctions()).isNotEmpty(); assertThat(ks.getAggregates()).isNotEmpty(); } @@ -177,7 +184,7 @@ private static File getScriptFile() { logbackTestUrl); } File resourcesDir = new File(logbackTestUrl.getFile()).getParentFile(); - File scriptsDir = new File(resourcesDir, isDse ? "DescribeIT/dse" : "DescribeIT/oss"); + File scriptsDir = new File(resourcesDir, scriptFileForBackend.get(CCM_RULE.getDistribution())); LOG.debug("Looking for a matching script in directory {}", scriptsDir); File[] candidates = scriptsDir.listFiles(); @@ -204,8 +211,7 @@ private static File getScriptFile() { .as("Could not find create script with version <= %s in %s", serverVersion, scriptsDir) .isNotNull(); - LOG.info( - "Using {} to test against {} {}", bestFile, isDse ? "DSE" : "Cassandra", serverVersion); + LOG.info("Using {} to test against {} {}", bestFile, CCM_RULE.getDistribution(), serverVersion); return bestFile; } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java index c7b51c040b5..8f5680ff41a 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java @@ -62,8 +62,9 @@ public void should_expose_node_metadata() { assertThat(node.getListenAddress().get().getAddress()).isEqualTo(connectAddress.getAddress()); assertThat(node.getDatacenter()).isEqualTo("dc1"); assertThat(node.getRack()).isEqualTo("r1"); - if (!CcmBridge.DSE_ENABLEMENT) { - // CcmBridge does not report accurate C* versions for DSE, only approximated values + if (CcmBridge.isDistributionOf(BackendType.CASSANDRA)) { + // CcmBridge does not report accurate C* versions for other distributions (e.g. DSE), only + // approximated values assertThat(node.getCassandraVersion()).isEqualTo(ccmRule.getCassandraVersion()); } assertThat(node.getState()).isSameAs(NodeState.UP); @@ -106,7 +107,7 @@ public void should_expose_dse_node_properties() { DseNodeProperties.DSE_WORKLOADS, DseNodeProperties.SERVER_ID); assertThat(node.getExtras().get(DseNodeProperties.DSE_VERSION)) - .isEqualTo(ccmRule.getDseVersion().get()); + .isEqualTo(ccmRule.getDistributionVersion()); assertThat(node.getExtras().get(DseNodeProperties.SERVER_ID)).isInstanceOf(String.class); assertThat(node.getExtras().get(DseNodeProperties.DSE_WORKLOADS)).isInstanceOf(Set.class); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java index 6f1dcb791c6..85fcfc02cdb 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java @@ -33,6 +33,7 @@ import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.google.common.collect.ImmutableList; @@ -54,8 +55,8 @@ public class SchemaChangesIT { static { CustomCcmRule.Builder builder = CustomCcmRule.builder(); - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> cass.nextStable().compareTo(Version.V4_0_0) >= 0)) { builder.withCassandraConfiguration("enable_materialized_views", true); } CCM_RULE = builder.build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java index 805b2d970cc..df5571974c1 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java @@ -335,11 +335,9 @@ public void should_exclude_virtual_keyspaces_from_token_map() { private void skipIfDse60() { // Special case: DSE 6.0 reports C* 4.0 but does not support virtual tables - if (ccmRule.getDseVersion().isPresent()) { - Version dseVersion = ccmRule.getDseVersion().get(); - if (dseVersion.compareTo(DSE_MIN_VIRTUAL_TABLES) < 0) { - throw new AssumptionViolatedException("DSE 6.0 does not support virtual tables"); - } + if (!ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(DSE_MIN_VIRTUAL_TABLES) >= 0)) { + throw new AssumptionViolatedException("DSE 6.0 does not support virtual tables"); } } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java index 3a418c73653..2eb898021ba 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java @@ -37,6 +37,7 @@ import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import io.reactivex.Flowable; import java.util.UUID; @@ -57,8 +58,8 @@ public class DeleteReactiveIT extends InventoryITBase { @ClassRule public static TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); private static CustomCcmRule.Builder configureCcm(CustomCcmRule.Builder builder) { - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> cass.nextStable().compareTo(Version.V4_0_0) >= 0)) { builder.withCassandraConfiguration("enable_sasi_indexes", true); } return builder; diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java index 9495003ae49..1bd899e4541 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java @@ -23,10 +23,10 @@ import com.datastax.oss.driver.api.mapper.annotations.Entity; import com.datastax.oss.driver.api.mapper.annotations.PartitionKey; import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.UUID; /** Factors common code for mapper tests that rely on a simple inventory model. */ @@ -93,13 +93,14 @@ protected static List createStatements(BaseCcmRule ccmRule, boolean requ return builder.build(); } - private static final Version MINIMUM_SASI_VERSION = Version.parse("3.4.0"); - private static final Version BROKEN_SASI_VERSION = Version.parse("6.8.0"); + private static final Version MINIMUM_SASI_VERSION = + Objects.requireNonNull(Version.parse("3.4.0")); + private static final Version BROKEN_SASI_VERSION = Objects.requireNonNull(Version.parse("6.8.0")); protected static boolean isSasiBroken(BaseCcmRule ccmRule) { - Optional dseVersion = ccmRule.getDseVersion(); // creating SASI indexes is broken in DSE 6.8.0 - return dseVersion.isPresent() && dseVersion.get().compareTo(BROKEN_SASI_VERSION) == 0; + return ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(BROKEN_SASI_VERSION) == 0); } protected static boolean supportsSASI(BaseCcmRule ccmRule) { diff --git a/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql b/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql new file mode 100644 index 00000000000..abc70728206 --- /dev/null +++ b/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql @@ -0,0 +1,186 @@ + +CREATE KEYSPACE ks_0 WITH replication = { 'class' : 'org.apache.cassandra.locator.SimpleStrategy', 'replication_factor': '1' } AND durable_writes = true; + +CREATE TYPE ks_0.btype ( + a text +); + +CREATE TYPE ks_0.xtype ( + d text +); + +CREATE TYPE ks_0.ztype ( + c text, + a int +); + +CREATE TYPE ks_0.ctype ( + z frozen, + x frozen +); + +CREATE TYPE ks_0.atype ( + c frozen +); + +CREATE TABLE ks_0.cyclist_mv ( + cid uuid, + age int, + birthday date, + country text, + name text, + PRIMARY KEY (cid) +) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE INDEX cyclist_by_country ON ks_0.cyclist_mv (country); + +CREATE TABLE ks_0.rank_by_year_and_name ( + race_year int, + race_name text, + rank int, + cyclist_name text, + PRIMARY KEY ((race_year, race_name), rank) +) WITH CLUSTERING ORDER BY (rank DESC) + AND additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE INDEX rrank ON ks_0.rank_by_year_and_name (rank); + +CREATE INDEX ryear ON ks_0.rank_by_year_and_name (race_year); + +CREATE TABLE ks_0.ztable ( + zkey text, + a frozen, + PRIMARY KEY (zkey) +) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.LeveledCompactionStrategy','max_threshold':'32','min_threshold':'4','sstable_size_in_mb':'95'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_a_age AS +SELECT * FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_age AS +SELECT + age, + cid, + birthday, + country, + name +FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = 'simple view' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_r_age AS +SELECT + age, + cid, + birthday, + country, + name +FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE FUNCTION ks_0.avgfinal(state tuple) + CALLED ON NULL INPUT + RETURNS double + LANGUAGE java + AS 'double r = 0; if (state.getInt(0) == 0) return null; r = state.getLong(1); r /= state.getInt(0); return Double.valueOf(r);'; + +CREATE FUNCTION ks_0.avgstate(state tuple,val int) + CALLED ON NULL INPUT + RETURNS tuple + LANGUAGE java + AS 'if (val !=null) { state.setInt(0, state.getInt(0)+1); state.setLong(1, state.getLong(1)+val.intValue()); } return state;'; + +CREATE AGGREGATE ks_0.average(int) + SFUNC avgstate + STYPE tuple + FINALFUNC avgfinal + INITCOND (0,0); + +CREATE AGGREGATE ks_0.mean(int) + SFUNC avgstate + STYPE tuple + FINALFUNC avgfinal + INITCOND (0,0); diff --git a/osgi-tests/README.md b/osgi-tests/README.md index 89ad0ba27c8..1ca6211d427 100644 --- a/osgi-tests/README.md +++ b/osgi-tests/README.md @@ -53,8 +53,8 @@ OSGi ones, you can do so as follows: You can pass the following system properties to your tests: 1. `ccm.version`: the CCM version to use -2. `ccm.dse`: whether to use DSE -3. `osgi.debug`: whether to enable remote debugging of the OSGi container (see +2. `ccm.distribution`: choose target backend type (e.g. DSE, HCD) +3. `osgi.debug`: whether to enable remote debugging of the OSGi container (see below). ## Debugging OSGi tests diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java index 8b140930870..ce4d9095361 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import java.util.List; import java.util.Objects; import net.jcip.annotations.GuardedBy; @@ -38,7 +39,7 @@ public class CcmStagedReactor extends AllConfinedStagedReactor { static { CcmBridge.Builder builder = CcmBridge.builder().withNodes(1); - if (CcmBridge.DSE_ENABLEMENT && CcmBridge.VERSION.compareTo(DSE_5_0) >= 0) { + if (CcmBridge.isDistributionOf(BackendType.DSE, (dist, cass) -> dist.compareTo(DSE_5_0) >= 0)) { builder.withDseWorkloads("graph"); } CCM_BRIDGE = builder.build(); @@ -54,11 +55,10 @@ public CcmStagedReactor(List containers, List m @Override public synchronized void beforeSuite() { if (!running) { - boolean dse = CCM_BRIDGE.getDseVersion().isPresent(); LOGGER.info( "Starting CCM, running {} version {}", - dse ? "DSE" : "Cassandra", - dse ? CCM_BRIDGE.getDseVersion().get() : CCM_BRIDGE.getCassandraVersion()); + CcmBridge.DISTRIBUTION, + CcmBridge.getDistributionVersion()); CCM_BRIDGE.create(); CCM_BRIDGE.start(); LOGGER.info("CCM started"); diff --git a/test-infra/revapi.json b/test-infra/revapi.json index 3cfbc8b5337..c75a98cb4af 100644 --- a/test-infra/revapi.json +++ b/test-infra/revapi.json @@ -171,6 +171,27 @@ "code": "java.method.removed", "old": "method void com.datastax.oss.driver.api.testinfra.ccm.CcmRule::reloadCore(int, java.lang.String, java.lang.String, boolean)", "justification": "Modifying the state of a globally shared CCM instance is dangerous" + }, + { + "code": "java.method.removed", + "old": "method java.util.Optional com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule::getDseVersion()", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" + }, + { + "code": "java.field.removed", + "old": "field com.datastax.oss.driver.api.testinfra.ccm.CcmBridge.DSE_ENABLEMENT", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" + }, + { + "code": "java.method.nowStatic", + "old": "method com.datastax.oss.driver.api.core.Version com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getCassandraVersion()", + "new": "method com.datastax.oss.driver.api.core.Version com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getCassandraVersion()", + "justification": "Previous and current implemntation do not relay on non-static fields" + }, + { + "code": "java.method.removed", + "old": "method java.util.Optional com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getDseVersion()", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" } ] } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java index 65210acd2a2..882cd55b948 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java @@ -22,7 +22,7 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.testinfra.CassandraResourceRule; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirementRule; -import java.util.Optional; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import org.junit.AssumptionViolatedException; import org.junit.runner.Description; import org.junit.runners.model.Statement; @@ -72,17 +72,29 @@ public void evaluate() { } } - public Version getCassandraVersion() { - return ccmBridge.getCassandraVersion(); + public BackendType getDistribution() { + return CcmBridge.DISTRIBUTION; + } + + public boolean isDistributionOf(BackendType type) { + return CcmBridge.isDistributionOf(type); + } + + public boolean isDistributionOf(BackendType type, CcmBridge.VersionComparator comparator) { + return CcmBridge.isDistributionOf(type, comparator); + } + + public Version getDistributionVersion() { + return CcmBridge.getDistributionVersion(); } - public Optional getDseVersion() { - return ccmBridge.getDseVersion(); + public Version getCassandraVersion() { + return CcmBridge.getCassandraVersion(); } @Override public ProtocolVersion getHighestProtocolVersion() { - if (ccmBridge.getCassandraVersion().compareTo(Version.V2_2_0) >= 0) { + if (CcmBridge.getCassandraVersion().compareTo(Version.V2_2_0) >= 0) { return DefaultProtocolVersion.V4; } else { return DefaultProtocolVersion.V3; diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 5b0c114a5fe..f0ce6bc5b0e 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.api.testinfra.ccm; import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.base.Joiner; import com.datastax.oss.driver.shaded.guava.common.io.Resources; import java.io.File; @@ -54,6 +55,9 @@ public class CcmBridge implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(CcmBridge.class); + public static BackendType DISTRIBUTION = + BackendType.valueOf( + System.getProperty("ccm.distribution", BackendType.CASSANDRA.name()).toUpperCase()); public static final Version VERSION = Objects.requireNonNull(Version.parse(System.getProperty("ccm.version", "4.0.0"))); @@ -61,8 +65,6 @@ public class CcmBridge implements AutoCloseable { public static final String BRANCH = System.getProperty("ccm.branch"); - public static final Boolean DSE_ENABLEMENT = Boolean.getBoolean("ccm.dse"); - public static final String CLUSTER_NAME = "ccm_1"; public static final String DEFAULT_CLIENT_TRUSTSTORE_PASSWORD = "fakePasswordForTests"; @@ -101,22 +103,21 @@ public class CcmBridge implements AutoCloseable { createTempStore(DEFAULT_SERVER_LOCALHOST_KEYSTORE_PATH); // major DSE versions - private static final Version V6_0_0 = Version.parse("6.0.0"); - private static final Version V5_1_0 = Version.parse("5.1.0"); - private static final Version V5_0_0 = Version.parse("5.0.0"); + public static final Version V6_0_0 = Version.parse("6.0.0"); + public static final Version V5_1_0 = Version.parse("5.1.0"); + public static final Version V5_0_0 = Version.parse("5.0.0"); // mapped C* versions from DSE versions - private static final Version V4_0_0 = Version.parse("4.0.0"); - private static final Version V3_10 = Version.parse("3.10"); - private static final Version V3_0_15 = Version.parse("3.0.15"); - private static final Version V2_1_19 = Version.parse("2.1.19"); + public static final Version V4_0_0 = Version.parse("4.0.0"); + public static final Version V3_10 = Version.parse("3.10"); + public static final Version V3_0_15 = Version.parse("3.0.15"); + public static final Version V2_1_19 = Version.parse("2.1.19"); + + // mapped C* versions from HCD versions + public static final Version V4_0_11 = Version.parse("4.0.11"); static { - if (DSE_ENABLEMENT) { - LOG.info("CCM Bridge configured with DSE version {}", VERSION); - } else { - LOG.info("CCM Bridge configured with Apache Cassandra version {}", VERSION); - } + LOG.info("CCM Bridge configured with {} version {}", DISTRIBUTION.getFriendlyName(), VERSION); } private final int[] nodes; @@ -175,25 +176,24 @@ private static boolean isWindows() { return System.getProperty("os.name", "").toLowerCase(Locale.US).contains("win"); } - public Optional getDseVersion() { - return DSE_ENABLEMENT ? Optional.of(VERSION) : Optional.empty(); + public static boolean isDistributionOf(BackendType type) { + return DISTRIBUTION == type; + } + + public static boolean isDistributionOf(BackendType type, VersionComparator comparator) { + return isDistributionOf(type) + && comparator.accept(getDistributionVersion(), getCassandraVersion()); + } + + public static Version getDistributionVersion() { + return VERSION; } - public Version getCassandraVersion() { - if (!DSE_ENABLEMENT) { + public static Version getCassandraVersion() { + if (isDistributionOf(BackendType.CASSANDRA)) { return VERSION; - } else { - Version stableVersion = VERSION.nextStable(); - if (stableVersion.compareTo(V6_0_0) >= 0) { - return V4_0_0; - } else if (stableVersion.compareTo(V5_1_0) >= 0) { - return V3_10; - } else if (stableVersion.compareTo(V5_0_0) >= 0) { - return V3_0_15; - } else { - return V2_1_19; - } } + return DistributionCassandraVersions.getCassandraVersion(DISTRIBUTION, VERSION); } private String getCcmVersionString(Version version) { @@ -225,9 +225,7 @@ public void create() { } else { createOptions.add("-v " + getCcmVersionString(VERSION)); } - if (DSE_ENABLEMENT) { - createOptions.add("--dse"); - } + createOptions.addAll(Arrays.asList(DISTRIBUTION.getCcmOptions())); execute( "create", CLUSTER_NAME, @@ -252,7 +250,7 @@ public void create() { // If we're dealing with anything more recent than 2.2 explicitly enable UDF... but run it // through our conversion process to make // sure more recent versions don't have a problem. - if (cassandraVersion.compareTo(Version.V2_2_0) >= 0) { + if (cassandraVersion.compareTo(Version.V2_2_0) >= 0 || isDistributionOf(BackendType.HCD)) { String originalKey = "enable_user_defined_functions"; Object originalValue = "true"; execute( @@ -264,7 +262,7 @@ public void create() { } // Note that we aren't performing any substitution on DSE key/value props (at least for now) - if (DSE_ENABLEMENT) { + if (isDistributionOf(BackendType.DSE)) { for (Map.Entry conf : dseConfiguration.entrySet()) { execute("updatedseconf", String.format("%s:%s", conf.getKey(), conf.getValue())); } @@ -338,11 +336,10 @@ public void stop(int n) { } public void add(int n, String dc) { - if (getDseVersion().isPresent()) { - execute("add", "-i", ipPrefix + n, "-d", dc, "node" + n, "--dse"); - } else { - execute("add", "-i", ipPrefix + n, "-d", dc, "node" + n); - } + List addOptions = new ArrayList<>(); + addOptions.addAll(Arrays.asList("add", "-i", ipPrefix + n, "-d", dc, "node" + n)); + addOptions.addAll(Arrays.asList(DISTRIBUTION.getCcmOptions())); + execute(addOptions.toArray(new String[0])); start(n); } @@ -475,11 +472,11 @@ private Optional overrideJvmVersionForDseWorkloads() { return Optional.empty(); } - if (!DSE_ENABLEMENT || !getDseVersion().isPresent()) { + if (!isDistributionOf(BackendType.DSE)) { return Optional.empty(); } - if (getDseVersion().get().compareTo(Version.V6_9_0) >= 0) { + if (getDistributionVersion().compareTo(Version.V6_9_0) >= 0) { // DSE 6.9.0 supports only JVM 11 onwards (also with graph workload) return Optional.empty(); } @@ -641,4 +638,8 @@ public CcmBridge build() { dseWorkloads); } } + + public interface VersionComparator { + boolean accept(Version distribution, Version cassandra); + } } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java index ac2507cec53..0819f785446 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java @@ -18,18 +18,20 @@ package com.datastax.oss.driver.api.testinfra.ccm; import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; /** @see CcmRule */ @SuppressWarnings("unused") public class DefaultCcmBridgeBuilderCustomizer { public static CcmBridge.Builder configureBuilder(CcmBridge.Builder builder) { - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.nextStable().compareTo(Version.V4_0_0) >= 0) + || CcmBridge.isDistributionOf(BackendType.HCD)) { builder.withCassandraConfiguration("enable_materialized_views", true); builder.withCassandraConfiguration("enable_sasi_indexes", true); } - if (CcmBridge.VERSION.nextStable().compareTo(Version.V3_0_0) >= 0) { + if (CcmBridge.getDistributionVersion().nextStable().compareTo(Version.V3_0_0) >= 0) { builder.withJvmArgs("-Dcassandra.superuser_setup_delay_ms=0"); builder.withJvmArgs("-Dcassandra.skip_wait_for_gossip_to_settle=0"); builder.withCassandraConfiguration("num_tokens", "1"); diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java new file mode 100644 index 00000000000..9f7634d1b37 --- /dev/null +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.testinfra.ccm; + +import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSortedMap; +import java.util.HashMap; +import java.util.Map; + +/** Defines mapping of various distributions to shipped Apache Cassandra version. */ +public abstract class DistributionCassandraVersions { + private static final Map> mappings = + new HashMap<>(); + + static { + { + // DSE + ImmutableSortedMap dse = + ImmutableSortedMap.of( + Version.V1_0_0, CcmBridge.V2_1_19, + Version.V5_0_0, CcmBridge.V3_0_15, + CcmBridge.V5_1_0, CcmBridge.V3_10, + CcmBridge.V6_0_0, CcmBridge.V4_0_0); + mappings.put(BackendType.DSE, dse); + } + { + // HCD + ImmutableSortedMap hcd = + ImmutableSortedMap.of(Version.V1_0_0, CcmBridge.V4_0_11); + mappings.put(BackendType.HCD, hcd); + } + } + + public static Version getCassandraVersion(BackendType type, Version version) { + ImmutableSortedMap mapping = mappings.get(type); + if (mapping == null) { + return null; + } + return mapping.floorEntry(version).getValue(); + } +} diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java index 6c59e216602..343861571e0 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java @@ -41,7 +41,7 @@ public void evaluate() { } protected static BackendType getBackendType() { - return CcmBridge.DSE_ENABLEMENT ? BackendType.DSE : BackendType.CASSANDRA; + return CcmBridge.DISTRIBUTION; } protected static Version getVersion() { diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java index 1683dd86136..e0058ca324a 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java @@ -18,9 +18,9 @@ package com.datastax.oss.driver.api.testinfra.requirement; public enum BackendType { - CASSANDRA("C*"), - DSE("Dse"), - ; + CASSANDRA("Apache Cassandra"), + DSE("DSE"), + HCD("HCD"); final String friendlyName; @@ -31,4 +31,11 @@ public enum BackendType { public String getFriendlyName() { return friendlyName; } + + public String[] getCcmOptions() { + if (this == CASSANDRA) { + return new String[0]; + } + return new String[] {"--" + name().toLowerCase()}; + } } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java index 5396e5c6cc6..3b792374769 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java @@ -29,10 +29,11 @@ import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.testinfra.CassandraResourceRule; import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import java.util.Objects; -import java.util.Optional; import org.junit.rules.ExternalResource; /** @@ -154,14 +155,12 @@ protected void before() { Statement.SYNC); } if (graphName != null) { - Optional dseVersion = - (cassandraResource instanceof BaseCcmRule) - ? ((BaseCcmRule) cassandraResource).getDseVersion() - : Optional.empty(); - if (!dseVersion.isPresent()) { + BaseCcmRule rule = + (cassandraResource instanceof BaseCcmRule) ? ((BaseCcmRule) cassandraResource) : null; + if (rule == null || !CcmBridge.isDistributionOf(BackendType.DSE)) { throw new IllegalArgumentException("DseSessionRule should work with DSE."); } - if (dseVersion.get().compareTo(V6_8_0) >= 0) { + if (rule.getDistributionVersion().compareTo(V6_8_0) >= 0) { session() .execute( ScriptGraphStatement.newInstance( From e84378681aecdbb87696dc4b53cb6fd336c82b6b Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 23 Oct 2024 22:23:39 +0200 Subject: [PATCH 24/77] Fix TableMetadata.describe() when containing a vector column patch by Stefan Miklosovic; reviewed by Bret McGuire for CASSJAVA-2 --- .../internal/core/type/DefaultVectorType.java | 2 +- .../metadata/schema/TableMetadataTest.java | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java index 5915adc2fb3..c9180d44edc 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java @@ -60,7 +60,7 @@ public String getClassName() { @NonNull @Override public String asCql(boolean includeFrozen, boolean pretty) { - return String.format("'%s(%d)'", getClassName(), getDimensions()); + return String.format("vector<%s, %d>", getElementType().asCql(true, false), getDimensions()); } /* ============== General class implementation ============== */ diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java new file mode 100644 index 00000000000..03d63230992 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata.schema; + +import static com.datastax.oss.driver.api.core.CqlIdentifier.fromCql; +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.internal.core.type.DefaultVectorType; +import com.datastax.oss.driver.internal.core.type.PrimitiveType; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.protocol.internal.ProtocolConstants.DataType; +import com.google.common.collect.ImmutableList; +import java.util.UUID; +import org.junit.Test; + +public class TableMetadataTest { + + /** Tests CASSJAVA-2 */ + @Test + public void should_describe_table_with_vector_correctly() { + TableMetadata tableMetadata = + new DefaultTableMetadata( + fromCql("ks"), + fromCql("tb"), + UUID.randomUUID(), + false, + false, + ImmutableList.of( + new DefaultColumnMetadata( + fromCql("ks"), + fromCql("ks"), + fromCql("tb"), + new PrimitiveType(DataType.ASCII), + false)), + ImmutableMap.of(), + ImmutableMap.of( + fromCql("a"), + new DefaultColumnMetadata( + fromCql("ks"), + fromCql("ks"), + fromCql("tb"), + new DefaultVectorType(new PrimitiveType(DataType.INT), 3), + false)), + ImmutableMap.of(), + ImmutableMap.of()); + + String describe1 = tableMetadata.describe(true); + + assertThat(describe1).contains("vector,"); + } +} From 62cea5a5f34d5cc6ef335f99829d0ae3cf9cf396 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 6 Nov 2024 17:47:30 -0600 Subject: [PATCH 25/77] Move Apache Cassandra 5.x off of beta1 and remove some older Apache Cassandra versions. patch by Bret McGuire; reviewed by Bret McGuire for CASSJAVA-54 --- Jenkinsfile-datastax | 52 +++++++++++++------------------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index af3faafee20..cd48f325a29 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -268,13 +268,9 @@ pipeline { ''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION', - choices: ['2.1', // Legacy Apache CassandraⓇ - '2.2', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Previous Apache CassandraⓇ - '4.0', // Previous Apache CassandraⓇ - '4.1', // Current Apache CassandraⓇ - '5.0-beta1', // Development Apache CassandraⓇ + choices: ['4.0', // Previous Apache CassandraⓇ + '4.1', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Long Term Support DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise @@ -292,22 +288,6 @@ pipeline { Choice Description - - 2.1 - Apache Cassandra® v2.1.x - - - 2.2 - Apache Cassandra® v2.2.x - - - 3.0 - Apache Cassandra® v3.0.x - - - 3.11 - Apache Cassandra® v3.11.x - 4.0 Apache Cassandra® v4.0.x @@ -316,6 +296,10 @@ pipeline { 4.1 Apache Cassandra® v4.1.x + + 5.0 + Apache Cassandra® v5.0.x + dse-4.8.16 DataStax Enterprise v4.8.x (END OF SERVICE LIFE) @@ -435,13 +419,10 @@ pipeline { // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ # Every weekend (Saturday, Sunday) around 2:00 AM - ### JDK8 tests against 2.1, 3.0, 4.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 - H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 + H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=4.0 4.1 5.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon - ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 - ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=4.1 5.0 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=4.1 5.0 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -475,8 +456,8 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ + values '4.0', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-6.8.30', // Current DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // Current DataStax HCD @@ -585,12 +566,9 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '2.1', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Previous Apache CassandraⓇ - '4.0', // Previous Apache CassandraⓇ - '4.1', // Current Apache CassandraⓇ - '5.0-beta1', // Development Apache CassandraⓇ + values '4.0', // Previous Apache CassandraⓇ + '4.1', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Last EOSL DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise From a322ca265654605123f4d7b889ad736c114f0c7e Mon Sep 17 00:00:00 2001 From: Jeremy Hanna Date: Wed, 27 Nov 2024 12:41:02 -0600 Subject: [PATCH 26/77] Update link to Jira to be CASSJAVA Updating the link to Jira. Previously we had a component in the CASSANDRA Jira project but now we have a project for each driver - in the case of Java, it's CASSJAVA. Added CASSJAVA to .asf.yaml patch by Jeremy Hanna; reviewed by Bret McGuire for CASSJAVA-61 --- .asf.yaml | 1 + README.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index ad58f536398..ac29efed9ff 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,3 +34,4 @@ github: projects: false autolink_jira: - CASSANDRA + - CASSJAVA diff --git a/README.md b/README.md index 2a30cb68c9a..b6e1cc337d8 100644 --- a/README.md +++ b/README.md @@ -75,13 +75,13 @@ See the [Cassandra error handling done right blog](https://www.datastax.com/blog * [Manual](manual/) * [API docs] -* Bug tracking: [JIRA]. Make sure to select the "Client/java-driver" component when filing new tickets! +* Bug tracking: [JIRA] * [Mailing list] * [Changelog] * [FAQ] [API docs]: https://docs.datastax.com/en/drivers/java/4.17 -[JIRA]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20CASSANDRA%20AND%20component%20%3D%20%22Client%2Fjava-driver%22%20ORDER%20BY%20key%20DESC +[JIRA]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20CASSJAVA%20ORDER%20BY%20key%20DESC [Mailing list]: https://groups.google.com/a/lists.datastax.com/forum/#!forum/java-driver-user [Changelog]: changelog/ [FAQ]: faq/ @@ -108,4 +108,4 @@ Apache Cassandra, Apache, Tomcat, Lucene, Solr, Hadoop, Spark, TinkerPop, and Ca trademarks of the [Apache Software Foundation](http://www.apache.org/) or its subsidiaries in Canada, the United States and/or other countries. -Binary artifacts of this product bundle Java Native Runtime libraries, which is available under the Eclipse Public License version 2.0. \ No newline at end of file +Binary artifacts of this product bundle Java Native Runtime libraries, which is available under the Eclipse Public License version 2.0. From 7bc085bdfb337b91255573bc3e130815e280954a Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Wed, 6 Nov 2024 08:31:36 +0100 Subject: [PATCH 27/77] Move DataStax shaded Guava module into Java driver patch by Lukasz Antoniak; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-52 --- bom/pom.xml | 10 +- core-shaded/pom.xml | 4 +- core/pom.xml | 4 +- distribution/src/assembly/binary-tarball.xml | 6 +- guava-shaded/pom.xml | 215 ++++++++++++++++++ guava-shaded/src/assembly/shaded-jar.xml | 48 ++++ ...graphicalComparatorHolderSubstitution.java | 39 ++++ .../UnsafeComparatorSubstitution.java | 25 ++ guava-shaded/src/main/javadoc/README.txt | 2 + manual/core/integration/README.md | 2 +- mapper-processor/pom.xml | 4 +- osgi-tests/pom.xml | 4 +- .../internal/osgi/support/BundleOptions.java | 2 +- pom.xml | 3 +- query-builder/pom.xml | 4 +- 15 files changed, 351 insertions(+), 21 deletions(-) create mode 100644 guava-shaded/pom.xml create mode 100644 guava-shaded/src/assembly/shaded-jar.xml create mode 100644 guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java create mode 100644 guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java create mode 100644 guava-shaded/src/main/javadoc/README.txt diff --git a/bom/pom.xml b/bom/pom.xml index 96b7a6ceb18..08f212f6157 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -55,6 +55,11 @@ java-driver-query-builder 4.18.2-SNAPSHOT + + org.apache.cassandra + java-driver-guava-shaded + 4.18.2-SNAPSHOT + org.apache.cassandra java-driver-test-infra @@ -75,11 +80,6 @@ native-protocol 1.5.1 - - com.datastax.oss - java-driver-shaded-guava - 25.1-jre-graal-sub-1 - diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 6c139aab127..9a708beb2a7 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -57,8 +57,8 @@ native-protocol - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.typesafe diff --git a/core/pom.xml b/core/pom.xml index 33688754f1b..2a48e8bf9ce 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -49,8 +49,8 @@ netty-handler - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.typesafe diff --git a/distribution/src/assembly/binary-tarball.xml b/distribution/src/assembly/binary-tarball.xml index 0d025fafb2c..b6294a25340 100644 --- a/distribution/src/assembly/binary-tarball.xml +++ b/distribution/src/assembly/binary-tarball.xml @@ -66,8 +66,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-mapper-runtime org.apache.cassandra:java-driver-mapper-processor + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations @@ -91,8 +91,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-query-builder org.apache.cassandra:java-driver-mapper-processor + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations @@ -116,8 +116,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-query-builder org.apache.cassandra:java-driver-mapper-runtime + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml new file mode 100644 index 00000000000..9854fcc48ba --- /dev/null +++ b/guava-shaded/pom.xml @@ -0,0 +1,215 @@ + + + + 4.0.0 + + org.apache.cassandra + java-driver-parent + 4.18.2-SNAPSHOT + + java-driver-guava-shaded + Apache Cassandra Java Driver - guava shaded dep + Shaded Guava artifact for use in the Java driver for Apache Cassandra® + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.j2objc + j2objc-annotations + + + org.codehaus.mojo + animal-sniffer-annotations + + + + + org.graalvm.nativeimage + svm + 20.0.0 + provided + + + + + + maven-shade-plugin + + + shade-guava-dependency + package + + shade + + + + + org.apache.cassandra:java-driver-guava-shaded + com.google.guava:guava + + + + + com.google + com.datastax.oss.driver.shaded.guava + + + + + com.google.guava:* + + META-INF/** + + + + true + true + + + + + + maven-clean-plugin + + + clean-classes + package + + clean + + + ${project.build.outputDirectory} + + + + + + maven-dependency-plugin + + + unpack-shaded-classes + package + + unpack + + + ${project.build.outputDirectory} + + + org.apache.cassandra + java-driver-guava-shaded + ${project.version} + jar + + + + + + + + org.apache.felix + maven-bundle-plugin + + 3.5.0 + true + + + generate-shaded-manifest + package + + manifest + + + + com.datastax.oss.driver.shaded.guava + !com.datastax.oss.driver.shaded.guava.errorprone.*, !org.checkerframework.*, * + javax.annotation.*;resolution:=optional;version="[3.0,4)", javax.crypto.*;resolution:=optional, sun.misc.*;resolution:=optional, !com.oracle.svm.*, !com.datastax.oss.driver.shaded.guava.errorprone.*, !org.checkerframework.*, * + + + + + + + maven-assembly-plugin + + + generate-final-shaded-jar + package + + single + + + + + ${project.build.outputDirectory}/META-INF/MANIFEST.MF + + + src/assembly/shaded-jar.xml + + + false + + + + + + maven-jar-plugin + + + empty-javadoc-jar + + jar + + + javadoc + ${basedir}/src/main/javadoc + + + + + + org.revapi + revapi-maven-plugin + + true + + + + + diff --git a/guava-shaded/src/assembly/shaded-jar.xml b/guava-shaded/src/assembly/shaded-jar.xml new file mode 100644 index 00000000000..d762a27b20f --- /dev/null +++ b/guava-shaded/src/assembly/shaded-jar.xml @@ -0,0 +1,48 @@ + + + + shaded-jar + + jar + + false + + + + ${project.build.outputDirectory} + + META-INF/maven/org.apache.cassandra/java-driver-guava-shaded/pom.xml + + + + + + + + ${project.basedir}/dependency-reduced-pom.xml + META-INF/maven/org.apache.cassandra/java-driver-guava-shaded + pom.xml + + + diff --git a/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java b/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java new file mode 100644 index 00000000000..95e9c70cdbc --- /dev/null +++ b/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.common.primitives; + +import com.oracle.svm.core.annotate.Alias; +import com.oracle.svm.core.annotate.RecomputeFieldValue; +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; +import java.util.Comparator; + +@TargetClass(UnsignedBytes.LexicographicalComparatorHolder.class) +final class LexicographicalComparatorHolderSubstitution { + + @Alias + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.FromAlias) + static Comparator BEST_COMPARATOR = UnsignedBytes.lexicographicalComparatorJavaImpl(); + + /* All known cases should be covered by the field substitution above... keeping this only + * for sake of completeness */ + @Substitute + static Comparator getBestComparator() { + return UnsignedBytes.lexicographicalComparatorJavaImpl(); + } +} diff --git a/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java b/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java new file mode 100644 index 00000000000..549de0b5c02 --- /dev/null +++ b/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.common.primitives; + +import com.oracle.svm.core.annotate.Delete; +import com.oracle.svm.core.annotate.TargetClass; + +@TargetClass(UnsignedBytes.LexicographicalComparatorHolder.UnsafeComparator.class) +@Delete +final class UnsafeComparatorSubstitution {} diff --git a/guava-shaded/src/main/javadoc/README.txt b/guava-shaded/src/main/javadoc/README.txt new file mode 100644 index 00000000000..57f82b2a265 --- /dev/null +++ b/guava-shaded/src/main/javadoc/README.txt @@ -0,0 +1,2 @@ +This empty JAR is generated for compliance with Maven Central rules. Please refer to the original +Guava API docs. \ No newline at end of file diff --git a/manual/core/integration/README.md b/manual/core/integration/README.md index 2dfc0155c63..f2a96160bce 100644 --- a/manual/core/integration/README.md +++ b/manual/core/integration/README.md @@ -671,7 +671,7 @@ The remaining core driver dependencies are the only ones that are truly mandator * the [native protocol](https://github.com/datastax/native-protocol) layer. This is essentially part of the driver code, but was externalized for reuse in other projects; -* `java-driver-shaded-guava`, a shaded version of [Guava](https://github.com/google/guava). It is +* `java-driver-guava-shaded`, a shaded version of [Guava](https://github.com/google/guava). It is relocated to a different package, and only used by internal driver code, so it should be completely transparent to third-party code; * the [SLF4J](https://www.slf4j.org/) API for [logging](../logging/). diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 61906f41987..6588f17d5f7 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -44,8 +44,8 @@ java-driver-mapper-runtime - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.squareup diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 5947aff1bc5..f0e66b656ca 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -71,8 +71,8 @@ logback-classic - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded org.xerial.snappy diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java index 536a6d96c77..3e6171ca530 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java @@ -35,7 +35,7 @@ public class BundleOptions { public static CompositeOption commonBundles() { return () -> options( - mavenBundle("com.datastax.oss", "java-driver-shaded-guava").versionAsInProject(), + mavenBundle("org.apache.cassandra", "java-driver-guava-shaded").versionAsInProject(), mavenBundle("io.dropwizard.metrics", "metrics-core").versionAsInProject(), mavenBundle("org.slf4j", "slf4j-api").versionAsInProject(), mavenBundle("org.hdrhistogram", "HdrHistogram").versionAsInProject(), diff --git a/pom.xml b/pom.xml index 94311719e5f..620cf1db4bb 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ mapper-processor metrics/micrometer metrics/microprofile + guava-shaded test-infra integration-tests osgi-tests @@ -110,7 +111,7 @@ ${netty.version} - + com.google.guava guava 25.1-jre diff --git a/query-builder/pom.xml b/query-builder/pom.xml index bae0e0c6ca0..4e09a10e584 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -45,8 +45,8 @@ java-driver-core - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.github.stephenc.jcip From 7ca013fbd5f1ec589df52ef0e6441986f07c11ff Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Sat, 22 Apr 2023 16:13:15 -0700 Subject: [PATCH 28/77] JAVA-3057 Allow decoding a UDT that has more fields than expected patch by Ammar Khaku; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1635 --- .../internal/core/type/codec/UdtCodec.java | 10 ++- .../core/type/codec/UdtCodecTest.java | 24 +++--- .../internal/core/type/codec/UdtCodecIT.java | 77 +++++++++++++++++++ 3 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java index f5177e63b5e..5d0a379f761 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java @@ -30,10 +30,14 @@ import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ThreadSafe public class UdtCodec implements TypeCodec { + private static final Logger LOG = LoggerFactory.getLogger(UdtCodec.class); + private final UserDefinedType cqlType; public UdtCodec(@NonNull UserDefinedType cqlType) { @@ -107,10 +111,8 @@ public UdtValue decode(@Nullable ByteBuffer bytes, @NonNull ProtocolVersion prot int i = 0; while (input.hasRemaining()) { if (i == cqlType.getFieldTypes().size()) { - throw new IllegalArgumentException( - String.format( - "Too many fields in encoded UDT value, expected %d", - cqlType.getFieldTypes().size())); + LOG.debug("Encountered unexpected fields when parsing codec {}", cqlType); + break; } int elementSize = input.getInt(); ByteBuffer element; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java index bf7c1e98b26..af94247f937 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java @@ -136,18 +136,18 @@ public void should_decode_udt() { } @Test - public void should_fail_to_decode_udt_when_too_many_fields() { - assertThatThrownBy( - () -> - decode( - "0x" - + ("00000004" + "00000001") - + "ffffffff" - + ("00000001" + "61") - // extra contents - + "ffffffff")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Too many fields in encoded UDT value, expected 3"); + public void should_decode_udt_when_too_many_fields() { + UdtValue udt = + decode( + "0x" + + ("00000004" + "00000001") + + "ffffffff" + + ("00000001" + "61") + // extra contents + + "ffffffff"); + assertThat(udt.getInt(0)).isEqualTo(1); + assertThat(udt.isNull(1)).isTrue(); + assertThat(udt.getString(2)).isEqualTo("a"); } /** Test for JAVA-2557. Ensures that the codec can decode null fields with any negative length. */ diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java new file mode 100644 index 00000000000..804a078bbe0 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.type.codec; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.data.UdtValue; +import com.datastax.oss.driver.api.core.type.UserDefinedType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.categories.ParallelizableTests; +import java.util.Objects; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@Category(ParallelizableTests.class) +public class UdtCodecIT { + + private CcmRule ccmRule = CcmRule.getInstance(); + + private SessionRule sessionRule = SessionRule.builder(ccmRule).build(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + + @Test + public void should_decoding_udt_be_backward_compatible() { + CqlSession session = sessionRule.session(); + session.execute("CREATE TYPE test_type_1 (a text, b int)"); + session.execute("CREATE TABLE test_table_1 (e int primary key, f frozen)"); + // insert a row using version 1 of the UDT schema + session.execute("INSERT INTO test_table_1(e, f) VALUES(1, {a: 'a', b: 1})"); + UserDefinedType udt = + session + .getMetadata() + .getKeyspace(sessionRule.keyspace()) + .flatMap(ks -> ks.getUserDefinedType("test_type_1")) + .orElseThrow(IllegalStateException::new); + TypeCodec oldCodec = session.getContext().getCodecRegistry().codecFor(udt); + // update UDT schema + session.execute("ALTER TYPE test_type_1 add i text"); + // insert a row using version 2 of the UDT schema + session.execute("INSERT INTO test_table_1(e, f) VALUES(2, {a: 'b', b: 2, i: 'b'})"); + Row row = + Objects.requireNonNull(session.execute("SELECT f FROM test_table_1 WHERE e = ?", 2).one()); + // Try to read new row with old codec. Using row.getUdtValue() would not cause any issues, + // because new codec will be automatically registered (using all 3 attributes). + // If application leverages generic row.get(String, Codec) method, data reading with old codec + // should + // be backward-compatible. + UdtValue value = Objects.requireNonNull((UdtValue) row.get("f", oldCodec)); + assertThat(value.getString("a")).isEqualTo("b"); + assertThat(value.getInt("b")).isEqualTo(2); + assertThatThrownBy(() -> value.getString("i")).hasMessage("i is not a field in this UDT"); + } +} From 6a8674f2db92668359196b5492753612b3844594 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 11 Nov 2024 11:49:20 -0600 Subject: [PATCH 29/77] CASSJAVA-55 Remove setting "Host" header for metadata requests. With some sysprops enabled this will actually be respected which completely borks Astra routing. patch by Bret McGuire; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-55 --- .../driver/internal/core/config/cloud/CloudConfigFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java index b6b2cccc466..817b3263d25 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java @@ -229,7 +229,6 @@ protected BufferedReader fetchProxyMetadata( HttpsURLConnection connection = (HttpsURLConnection) metadataServiceUrl.openConnection(); connection.setSSLSocketFactory(sslContext.getSocketFactory()); connection.setRequestMethod("GET"); - connection.setRequestProperty("host", "localhost"); return new BufferedReader( new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8)); } catch (ConnectException e) { From 7689c5a81e89bce5598d9976a041068a1f5e2a7f Mon Sep 17 00:00:00 2001 From: SiyaoIsHiding <113857408+SiyaoIsHiding@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:30:58 +0800 Subject: [PATCH 30/77] JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder patch by Jane He; reviewed by Mick Semb Wever and Bret McGuire for JAVA-3118 reference: #1931 --- manual/query_builder/select/README.md | 23 +++++ query-builder/revapi.json | 10 +++ .../api/querybuilder/select/Select.java | 11 +++ .../querybuilder/select/DefaultSelect.java | 86 +++++++++++++++++-- .../delete/DeleteSelectorTest.java | 11 +++ .../insert/RegularInsertTest.java | 7 ++ .../querybuilder/schema/AlterTableTest.java | 6 ++ .../querybuilder/schema/AlterTypeTest.java | 6 ++ .../querybuilder/schema/CreateTableTest.java | 9 ++ .../querybuilder/schema/CreateTypeTest.java | 9 ++ .../select/SelectOrderingTest.java | 20 +++++ .../select/SelectSelectorTest.java | 43 ++++++++++ 12 files changed, 233 insertions(+), 8 deletions(-) diff --git a/manual/query_builder/select/README.md b/manual/query_builder/select/README.md index 92c058608e7..0425423a402 100644 --- a/manual/query_builder/select/README.md +++ b/manual/query_builder/select/README.md @@ -387,6 +387,29 @@ selectFrom("sensor_data") // SELECT reading FROM sensor_data WHERE id=? ORDER BY date DESC ``` +Vector Search: + +```java + +import com.datastax.oss.driver.api.core.data.CqlVector; + +selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderByAnnOf("c1", CqlVector.newInstance(0.1, 0.2, 0.3)); +// SELECT * FROM foo WHERE k=1 ORDER BY c1 ANN OF [0.1, 0.2, 0.3] + +selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_cosine", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); +// SELECT comment,similarity_cosine(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1 +``` + Limits: ```java diff --git a/query-builder/revapi.json b/query-builder/revapi.json index 9d0163b487e..c4d8aa27212 100644 --- a/query-builder/revapi.json +++ b/query-builder/revapi.json @@ -2772,6 +2772,16 @@ "code": "java.method.addedToInterface", "new": "method com.datastax.oss.driver.api.querybuilder.update.UpdateStart com.datastax.oss.driver.api.querybuilder.update.UpdateStart::usingTtl(int)", "justification": "JAVA-2210: Add ability to set TTL for modification queries" + }, + { + "code": "java.method.addedToInterface", + "new": "method com.datastax.oss.driver.api.querybuilder.select.Select com.datastax.oss.driver.api.querybuilder.select.Select::orderByAnnOf(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector)", + "justification": "JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder" + }, + { + "code": "java.method.addedToInterface", + "new": "method com.datastax.oss.driver.api.querybuilder.select.Select com.datastax.oss.driver.api.querybuilder.select.Select::orderByAnnOf(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector)", + "justification": "JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder" } ] } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java index a22b45c35bd..159657989da 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.api.querybuilder.select; import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.BindMarker; import com.datastax.oss.driver.api.querybuilder.BuildableQuery; @@ -146,6 +147,16 @@ default Select orderBy(@NonNull String columnName, @NonNull ClusteringOrder orde return orderBy(CqlIdentifier.fromCql(columnName), order); } + /** + * Shortcut for {@link #orderByAnnOf(CqlIdentifier, CqlVector)}, adding an ORDER BY ... ANN OF ... + * clause + */ + @NonNull + Select orderByAnnOf(@NonNull String columnName, @NonNull CqlVector ann); + + /** Adds the ORDER BY ... ANN OF ... clause, usually used for vector search */ + @NonNull + Select orderByAnnOf(@NonNull CqlIdentifier columnId, @NonNull CqlVector ann); /** * Adds a LIMIT clause to this query with a literal value. * diff --git a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java index 86a2a07a3f2..5daf252a9eb 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java @@ -20,8 +20,10 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.SimpleStatementBuilder; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.BindMarker; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.relation.Relation; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.datastax.oss.driver.api.querybuilder.select.SelectFrom; @@ -49,6 +51,7 @@ public class DefaultSelect implements SelectFrom, Select { private final ImmutableList relations; private final ImmutableList groupByClauses; private final ImmutableMap orderings; + private final Ann ann; private final Object limit; private final Object perPartitionLimit; private final boolean allowsFiltering; @@ -65,6 +68,7 @@ public DefaultSelect(@Nullable CqlIdentifier keyspace, @NonNull CqlIdentifier ta ImmutableMap.of(), null, null, + null, false); } @@ -74,6 +78,8 @@ public DefaultSelect(@Nullable CqlIdentifier keyspace, @NonNull CqlIdentifier ta * @param selectors if it contains {@link AllSelector#INSTANCE}, that must be the only element. * This isn't re-checked because methods that call this constructor internally already do it, * make sure you do it yourself. + * @param ann Approximate nearest neighbor. ANN ordering does not support secondary ordering or + * ASC order. */ public DefaultSelect( @Nullable CqlIdentifier keyspace, @@ -84,6 +90,7 @@ public DefaultSelect( @NonNull ImmutableList relations, @NonNull ImmutableList groupByClauses, @NonNull ImmutableMap orderings, + @Nullable Ann ann, @Nullable Object limit, @Nullable Object perPartitionLimit, boolean allowsFiltering) { @@ -94,6 +101,9 @@ public DefaultSelect( || (limit instanceof Integer && (Integer) limit > 0) || limit instanceof BindMarker, "limit must be a strictly positive integer or a bind marker"); + Preconditions.checkArgument( + orderings.isEmpty() || ann == null, "ANN ordering does not support secondary ordering"); + this.ann = ann; this.keyspace = keyspace; this.table = table; this.isJson = isJson; @@ -117,6 +127,7 @@ public SelectFrom json() { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -134,6 +145,7 @@ public SelectFrom distinct() { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -193,6 +205,7 @@ public Select withSelectors(@NonNull ImmutableList newSelectors) { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -221,6 +234,7 @@ public Select withRelations(@NonNull ImmutableList newRelations) { newRelations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -249,6 +263,7 @@ public Select withGroupByClauses(@NonNull ImmutableList newGroupByClau relations, newGroupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -260,6 +275,18 @@ public Select orderBy(@NonNull CqlIdentifier columnId, @NonNull ClusteringOrder return withOrderings(ImmutableCollections.append(orderings, columnId, order)); } + @NonNull + @Override + public Select orderByAnnOf(@NonNull String columnName, @NonNull CqlVector ann) { + return withAnn(new Ann(CqlIdentifier.fromCql(columnName), ann)); + } + + @NonNull + @Override + public Select orderByAnnOf(@NonNull CqlIdentifier columnId, @NonNull CqlVector ann) { + return withAnn(new Ann(columnId, ann)); + } + @NonNull @Override public Select orderByIds(@NonNull Map newOrderings) { @@ -277,6 +304,24 @@ public Select withOrderings(@NonNull ImmutableMap entry : orderings.entrySet()) { - if (first) { - builder.append(" ORDER BY "); - first = false; - } else { - builder.append(","); + if (ann != null) { + builder.append(" ORDER BY ").append(this.ann.columnId.asCql(true)).append(" ANN OF "); + QueryBuilder.literal(ann.vector).appendTo(builder); + } else { + boolean first = true; + for (Map.Entry entry : orderings.entrySet()) { + if (first) { + builder.append(" ORDER BY "); + first = false; + } else { + builder.append(","); + } + builder.append(entry.getKey().asCql(true)).append(" ").append(entry.getValue().name()); } - builder.append(entry.getKey().asCql(true)).append(" ").append(entry.getValue().name()); } if (limit != null) { @@ -499,6 +554,11 @@ public Object getLimit() { return limit; } + @Nullable + public Ann getAnn() { + return ann; + } + @Nullable public Object getPerPartitionLimit() { return perPartitionLimit; @@ -512,4 +572,14 @@ public boolean allowsFiltering() { public String toString() { return asCql(); } + + public static class Ann { + private final CqlVector vector; + private final CqlIdentifier columnId; + + private Ann(CqlIdentifier columnId, CqlVector vector) { + this.vector = vector; + this.columnId = columnId; + } + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java index 23210971bc6..cce4cf51a10 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java @@ -22,6 +22,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.deleteFrom; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; +import com.datastax.oss.driver.api.core.data.CqlVector; import org.junit.Test; public class DeleteSelectorTest { @@ -34,6 +35,16 @@ public void should_generate_column_deletion() { .hasCql("DELETE v FROM ks.foo WHERE k=?"); } + @Test + public void should_generate_vector_deletion() { + assertThat( + deleteFrom("foo") + .column("v") + .whereColumn("k") + .isEqualTo(literal(CqlVector.newInstance(0.1, 0.2)))) + .hasCql("DELETE v FROM foo WHERE k=[0.1, 0.2]"); + } + @Test public void should_generate_field_deletion() { assertThat( diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java index 36133445b34..89c833ff1c6 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java @@ -23,6 +23,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static org.assertj.core.api.Assertions.catchThrowable; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.querybuilder.term.Term; import com.datastax.oss.driver.internal.querybuilder.insert.DefaultInsert; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; @@ -41,6 +42,12 @@ public void should_generate_column_assignments() { .hasCql("INSERT INTO foo (a,b) VALUES (?,?)"); } + @Test + public void should_generate_vector_literals() { + assertThat(insertInto("foo").value("a", literal(CqlVector.newInstance(0.1, 0.2, 0.3)))) + .hasCql("INSERT INTO foo (a) VALUES ([0.1, 0.2, 0.3])"); + } + @Test public void should_keep_last_assignment_if_column_listed_twice() { assertThat( diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java index 1567b0848cf..2c99b154b38 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java @@ -108,4 +108,10 @@ public void should_generate_alter_table_with_no_compression() { assertThat(alterTable("bar").withNoCompression()) .hasCql("ALTER TABLE bar WITH compression={'sstable_compression':''}"); } + + @Test + public void should_generate_alter_table_with_vector() { + assertThat(alterTable("bar").alterColumn("v", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("ALTER TABLE bar ALTER v TYPE vector"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java index 2becb9338f9..14bec0a6ce3 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java @@ -53,4 +53,10 @@ public void should_generate_alter_table_with_rename_three_columns() { assertThat(alterType("bar").renameField("x", "y").renameField("u", "v").renameField("b", "a")) .hasCql("ALTER TYPE bar RENAME x TO y AND u TO v AND b TO a"); } + + @Test + public void should_generate_alter_type_with_vector() { + assertThat(alterType("foo", "bar").alterField("vec", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("ALTER TYPE foo.bar ALTER vec TYPE vector"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java index 7a5542c51f0..15cd12c75eb 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java @@ -314,4 +314,13 @@ public void should_generate_create_table_time_window_compaction() { .hasCql( "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compaction={'class':'TimeWindowCompactionStrategy','compaction_window_size':10,'compaction_window_unit':'DAYS','timestamp_resolution':'MICROSECONDS','unsafe_aggressive_sstable_expiration':false}"); } + + @Test + public void should_generate_vector_column() { + assertThat( + createTable("foo") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("CREATE TABLE foo (k int PRIMARY KEY,v vector)"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java index d881a0500cb..f7c15788a0f 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java @@ -83,4 +83,13 @@ public void should_create_type_with_collections() { .withField("map", DataTypes.mapOf(DataTypes.INT, DataTypes.TEXT))) .hasCql("CREATE TYPE ks1.type (map map)"); } + + @Test + public void should_create_type_with_vector() { + assertThat( + createType("ks1", "type") + .withField("c1", DataTypes.INT) + .withField("vec", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("CREATE TYPE ks1.type (c1 int,vec vector)"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java index ff27fde4f8f..a9c618e9559 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java @@ -23,6 +23,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.querybuilder.relation.Relation; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import org.junit.Test; @@ -74,4 +75,23 @@ public void should_replace_previous_ordering() { .orderBy(ImmutableMap.of("c1", DESC, "c2", ASC))) .hasCql("SELECT * FROM foo WHERE k=1 ORDER BY c3 ASC,c1 DESC,c2 ASC"); } + + @Test + public void should_generate_ann_clause() { + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderByAnnOf("c1", CqlVector.newInstance(0.1, 0.2, 0.3))) + .hasCql("SELECT * FROM foo WHERE k=1 ORDER BY c1 ANN OF [0.1, 0.2, 0.3]"); + } + + @Test(expected = IllegalArgumentException.class) + public void should_fail_when_provided_ann_with_other_orderings() { + selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderBy("c1", ASC) + .orderByAnnOf("c2", CqlVector.newInstance(0.1, 0.2, 0.3)); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java index dc7cc98c6cc..7e03627d4b7 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java @@ -22,6 +22,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.raw; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.core.type.codec.CodecNotFoundException; import com.datastax.oss.driver.api.querybuilder.CharsetCodec; @@ -230,6 +231,48 @@ public void should_generate_raw_selector() { .hasCql("SELECT bar,baz FROM foo"); } + @Test + public void should_generate_similarity_functions() { + Select similarity_cosine_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_cosine", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_cosine_clause) + .hasCql( + "SELECT comment,similarity_cosine(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + + Select similarity_euclidean_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_euclidean", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_euclidean_clause) + .hasCql( + "SELECT comment,similarity_euclidean(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + + Select similarity_dot_product_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_dot_product", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_dot_product_clause) + .hasCql( + "SELECT comment,similarity_dot_product(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + } + @Test public void should_alias_selectors() { assertThat(selectFrom("foo").column("bar").as("baz")).hasCql("SELECT bar AS baz FROM foo"); From 8c1009959e988d9f7249151ad260f64aa6afed63 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 23 Dec 2024 08:20:42 +0100 Subject: [PATCH 31/77] Upgrade Guava to 33.3.1-jre patch by Lukasz Antoniak; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-53 --- .../internal/core/cql/reactive/TestSubscriber.java | 5 +++-- guava-shaded/pom.xml | 10 ++-------- .../dse/driver/api/mapper/reactive/TestSubscriber.java | 6 +++++- pom.xml | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java b/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java index aed7a4dfc8e..652155e5309 100644 --- a/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java +++ b/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java @@ -81,7 +81,8 @@ public List getElements() { } public void awaitTermination() { - Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES); - if (latch.getCount() > 0) fail("subscriber not terminated"); + if (!Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES)) { + fail("subscriber not terminated"); + } } } diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 9854fcc48ba..f480f9258cc 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -45,14 +45,6 @@ com.google.errorprone error_prone_annotations - - com.google.j2objc - j2objc-annotations - - - org.codehaus.mojo - animal-sniffer-annotations - @@ -78,6 +70,8 @@ org.apache.cassandra:java-driver-guava-shaded com.google.guava:guava + com.google.guava:failureaccess + com.google.j2objc:j2objc-annotations diff --git a/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java b/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java index 6f23cfca98a..6886b9a7622 100644 --- a/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java +++ b/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java @@ -17,6 +17,8 @@ */ package com.datastax.dse.driver.api.mapper.reactive; +import static org.assertj.core.api.Fail.fail; + import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -70,6 +72,8 @@ public List getElements() { } public void awaitTermination() { - Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES); + if (!Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES)) { + fail("subscriber not terminated"); + } } } diff --git a/pom.xml b/pom.xml index 620cf1db4bb..c61e6485fd3 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ com.google.guava guava - 25.1-jre + 33.3.1-jre com.typesafe From 75a269d04d49630032be6afedae2ded0c4334e42 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 30 Dec 2024 07:18:06 +0100 Subject: [PATCH 32/77] Do not always cleanup Guava shaded module before packaging --- guava-shaded/pom.xml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index f480f9258cc..6a22663e956 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -94,21 +94,6 @@ - - maven-clean-plugin - - - clean-classes - package - - clean - - - ${project.build.outputDirectory} - - - - maven-dependency-plugin From 01671d99247bdc783c832c128a8570ba846875c4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 10 Jan 2025 16:05:42 +0100 Subject: [PATCH 33/77] Revert "Do not always cleanup Guava shaded module before packaging" This reverts commit 5be52ec1a8d014c81566180c731b828a591082da. --- guava-shaded/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 6a22663e956..f480f9258cc 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -94,6 +94,21 @@ + + maven-clean-plugin + + + clean-classes + package + + clean + + + ${project.build.outputDirectory} + + + + maven-dependency-plugin From 342e2dcf47afab238f357ac2afde65b079ce6b79 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 10 Jan 2025 16:16:26 +0100 Subject: [PATCH 34/77] Conditionally compile shaded Guava module --- guava-shaded/pom.xml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index f480f9258cc..ca8f0161b04 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -56,6 +56,32 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + 1.12 + + + regex-property + + regex-property + + + maven.main.skip + ${java.version} + ^(?!1.8).+ + true + false + + + + maven-shade-plugin @@ -95,6 +121,12 @@ + maven-clean-plugin From 2e0c44c020819c928730e1aac812bf2f475d8256 Mon Sep 17 00:00:00 2001 From: SiyaoIsHiding <113857408+SiyaoIsHiding@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:38:52 +0800 Subject: [PATCH 35/77] JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Jane He; reviewed by Bret McGuire and João Reis reference: https://github.com/apache/cassandra-java-driver/pull/1952 --- core/revapi.json | 297 ++++++++++++++++++ .../oss/driver/api/core/data/CqlVector.java | 72 ++++- .../driver/api/core/data/GettableById.java | 2 +- .../driver/api/core/data/GettableByIndex.java | 3 +- .../driver/api/core/data/GettableByName.java | 2 +- .../driver/api/core/data/SettableById.java | 2 +- .../driver/api/core/data/SettableByIndex.java | 2 +- .../driver/api/core/data/SettableByName.java | 2 +- .../oss/driver/api/core/type/DataTypes.java | 19 +- .../driver/api/core/type/codec/TypeCodec.java | 6 + .../api/core/type/codec/TypeCodecs.java | 4 +- .../api/core/type/reflect/GenericType.java | 6 +- .../parsing/DataTypeClassNameParser.java | 8 + .../internal/core/type/DefaultVectorType.java | 2 +- .../internal/core/type/codec/BigIntCodec.java | 7 + .../core/type/codec/BooleanCodec.java | 7 + .../internal/core/type/codec/DoubleCodec.java | 7 + .../internal/core/type/codec/FloatCodec.java | 7 + .../internal/core/type/codec/IntCodec.java | 7 + .../core/type/codec/TimestampCodec.java | 7 + .../internal/core/type/codec/UuidCodec.java | 7 + .../internal/core/type/codec/VectorCodec.java | 86 +++-- .../extras/time/TimestampMillisCodec.java | 7 + .../codec/registry/CachingCodecRegistry.java | 12 +- .../internal/core/type/util/VIntCoding.java | 77 ++++- .../driver/api/core/data/CqlVectorTest.java | 167 +++++----- .../core/type/codec/VectorCodecTest.java | 265 ++++++++++++---- ...CachingCodecRegistryTestDataProviders.java | 20 ++ .../core/type/util/VIntCodingTest.java | 86 +++++ .../oss/driver/core/data/DataTypeIT.java | 38 ++- 30 files changed, 1002 insertions(+), 232 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java diff --git a/core/revapi.json b/core/revapi.json index 1c875895d6c..5aa46a3ccad 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -7089,6 +7089,303 @@ "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeChanged", + "old": "method T com.datastax.oss.driver.api.core.data.CqlVector::get(int)", + "new": "method T com.datastax.oss.driver.api.core.data.CqlVector::get(int)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.Iterator com.datastax.oss.driver.api.core.data.CqlVector::iterator()", + "new": "method java.util.Iterator com.datastax.oss.driver.api.core.data.CqlVector::iterator()", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===V[]===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===V[]===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===java.util.List===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===java.util.List===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter T com.datastax.oss.driver.api.core.data.CqlVector::set(int, ===T===)", + "new": "parameter T com.datastax.oss.driver.api.core.data.CqlVector::set(int, ===T===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeChanged", + "old": "method T com.datastax.oss.driver.api.core.data.CqlVector::set(int, T)", + "new": "method T com.datastax.oss.driver.api.core.data.CqlVector::set(int, T)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.Spliterator java.lang.Iterable::spliterator() @ com.datastax.oss.driver.api.core.data.CqlVector", + "new": "method java.util.Spliterator java.lang.Iterable::spliterator() @ com.datastax.oss.driver.api.core.data.CqlVector", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.stream.Stream com.datastax.oss.driver.api.core.data.CqlVector::stream()", + "new": "method java.util.stream.Stream com.datastax.oss.driver.api.core.data.CqlVector::stream()", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::subVector(int, int)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::subVector(int, int)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.class.noLongerImplementsInterface", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "interface": "java.lang.Iterable", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.class.superTypeTypeParametersChanged", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "oldSuperType": "java.lang.Iterable", + "newSuperType": "java.lang.Iterable", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===com.datastax.oss.driver.api.core.type.reflect.GenericType===)", + "new": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===com.datastax.oss.driver.api.core.type.reflect.GenericType===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" } ] } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java index 911b6187f6d..8089d551750 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java @@ -18,11 +18,10 @@ package com.datastax.oss.driver.api.core.data; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.internal.core.type.codec.ParseUtils; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.base.Predicates; -import com.datastax.oss.driver.shaded.guava.common.base.Splitter; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; -import com.datastax.oss.driver.shaded.guava.common.collect.Streams; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.io.InvalidObjectException; @@ -35,7 +34,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -52,7 +50,7 @@ * where possible we've tried to make the API of this class similar to the equivalent methods on * {@link List}. */ -public class CqlVector implements Iterable, Serializable { +public class CqlVector implements Iterable, Serializable { /** * Create a new CqlVector containing the specified values. @@ -60,7 +58,7 @@ public class CqlVector implements Iterable, Serializable { * @param vals the collection of values to wrap. * @return a CqlVector wrapping those values */ - public static CqlVector newInstance(V... vals) { + public static CqlVector newInstance(V... vals) { // Note that Array.asList() guarantees the return of an array which implements RandomAccess return new CqlVector(Arrays.asList(vals)); @@ -73,29 +71,64 @@ public static CqlVector newInstance(V... vals) { * @param list the collection of values to wrap. * @return a CqlVector wrapping those values */ - public static CqlVector newInstance(List list) { + public static CqlVector newInstance(List list) { Preconditions.checkArgument(list != null, "Input list should not be null"); return new CqlVector(list); } /** - * Create a new CqlVector instance from the specified string representation. Note that this method - * is intended to mirror {@link #toString()}; passing this method the output from a toString - * call on some CqlVector should return a CqlVector that is equal to the origin instance. + * Create a new CqlVector instance from the specified string representation. * * @param str a String representation of a CqlVector * @param subtypeCodec * @return a new CqlVector built from the String representation */ - public static CqlVector from( - @NonNull String str, @NonNull TypeCodec subtypeCodec) { + public static CqlVector from(@NonNull String str, @NonNull TypeCodec subtypeCodec) { Preconditions.checkArgument(str != null, "Cannot create CqlVector from null string"); Preconditions.checkArgument(!str.isEmpty(), "Cannot create CqlVector from empty string"); - ArrayList vals = - Streams.stream(Splitter.on(", ").split(str.substring(1, str.length() - 1))) - .map(subtypeCodec::parse) - .collect(Collectors.toCollection(ArrayList::new)); - return new CqlVector(vals); + if (str.equalsIgnoreCase("NULL")) return null; + + int idx = ParseUtils.skipSpaces(str, 0); + if (str.charAt(idx++) != '[') + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", at character %d expecting '[' but got '%c'", + str, idx, str.charAt(idx))); + + idx = ParseUtils.skipSpaces(str, idx); + + if (str.charAt(idx) == ']') { + return new CqlVector<>(new ArrayList<>()); + } + + List list = new ArrayList<>(); + while (idx < str.length()) { + int n; + try { + n = ParseUtils.skipCQLValue(str, idx); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", invalid CQL value at character %d", + str, idx), + e); + } + + list.add(subtypeCodec.parse(str.substring(idx, n))); + idx = n; + + idx = ParseUtils.skipSpaces(str, idx); + if (str.charAt(idx) == ']') return new CqlVector<>(list); + if (str.charAt(idx++) != ',') + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", at character %d expecting ',' but got '%c'", + str, idx, str.charAt(idx))); + + idx = ParseUtils.skipSpaces(str, idx); + } + throw new IllegalArgumentException( + String.format("Malformed vector value \"%s\", missing closing ']'", str)); } private final List list; @@ -194,6 +227,11 @@ public int hashCode() { return Objects.hash(list); } + /** + * The string representation of the vector. Elements, like strings, may not be properly quoted. + * + * @return the string representation + */ @Override public String toString() { return Iterables.toString(this.list); @@ -205,7 +243,7 @@ public String toString() { * * @param inner type of CqlVector, assume Number is always Serializable. */ - private static class SerializationProxy implements Serializable { + private static class SerializationProxy implements Serializable { private static final long serialVersionUID = 1; diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java index 0a24214b20a..8393bc9f758 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java @@ -531,7 +531,7 @@ default CqlDuration getCqlDuration(@NonNull CqlIdentifier id) { * @throws IllegalArgumentException if the id is invalid. */ @Nullable - default CqlVector getVector( + default CqlVector getVector( @NonNull CqlIdentifier id, @NonNull Class elementsClass) { return getVector(firstIndexOf(id), elementsClass); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java index 53541b0ac58..bb75bd9a2b4 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java @@ -446,8 +446,7 @@ default CqlDuration getCqlDuration(int i) { * @throws IndexOutOfBoundsException if the index is invalid. */ @Nullable - default CqlVector getVector( - int i, @NonNull Class elementsClass) { + default CqlVector getVector(int i, @NonNull Class elementsClass) { return get(i, GenericType.vectorOf(elementsClass)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java index ec3ee362ca8..b0a4660033b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java @@ -527,7 +527,7 @@ default CqlDuration getCqlDuration(@NonNull String name) { * @throws IllegalArgumentException if the name is invalid. */ @Nullable - default CqlVector getVector( + default CqlVector getVector( @NonNull String name, @NonNull Class elementsClass) { return getVector(firstIndexOf(name), elementsClass); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java index 8452446205e..0f5e3cd9daa 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java @@ -573,7 +573,7 @@ default SelfT setCqlDuration(@NonNull CqlIdentifier id, @Nullable CqlDuration v) */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( @NonNull CqlIdentifier id, @Nullable CqlVector v, @NonNull Class elementsClass) { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java index bb55db3adde..4ecdf647590 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java @@ -425,7 +425,7 @@ default SelfT setCqlDuration(int i, @Nullable CqlDuration v) { */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( int i, @Nullable CqlVector v, @NonNull Class elementsClass) { return set(i, v, GenericType.vectorOf(elementsClass)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java index c25a7074373..afe9ba59f64 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java @@ -572,7 +572,7 @@ default SelfT setCqlDuration(@NonNull String name, @Nullable CqlDuration v) { */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( @NonNull String name, @Nullable CqlVector v, @NonNull Class elementsClass) { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java index 3a341eaa5aa..492fc121c71 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java @@ -27,12 +27,10 @@ import com.datastax.oss.driver.internal.core.type.DefaultTupleType; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; import com.datastax.oss.driver.internal.core.type.PrimitiveType; -import com.datastax.oss.driver.shaded.guava.common.base.Splitter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.ProtocolConstants; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Arrays; -import java.util.List; /** Constants and factory methods to obtain data type instances. */ public class DataTypes { @@ -59,7 +57,6 @@ public class DataTypes { public static final DataType DURATION = new PrimitiveType(ProtocolConstants.DataType.DURATION); private static final DataTypeClassNameParser classNameParser = new DataTypeClassNameParser(); - private static final Splitter paramSplitter = Splitter.on(',').trimResults(); @NonNull public static DataType custom(@NonNull String className) { @@ -68,20 +65,8 @@ public static DataType custom(@NonNull String className) { if (className.equals("org.apache.cassandra.db.marshal.DurationType")) return DURATION; /* Vector support is currently implemented as a custom type but is also parameterized */ - if (className.startsWith(DefaultVectorType.VECTOR_CLASS_NAME)) { - List params = - paramSplitter.splitToList( - className.substring( - DefaultVectorType.VECTOR_CLASS_NAME.length() + 1, className.length() - 1)); - DataType subType = classNameParser.parse(params.get(0), AttachmentPoint.NONE); - int dimensions = Integer.parseInt(params.get(1)); - if (dimensions <= 0) { - throw new IllegalArgumentException( - String.format( - "Request to create vector of size %d, size must be positive", dimensions)); - } - return new DefaultVectorType(subType, dimensions); - } + if (className.startsWith(DefaultVectorType.VECTOR_CLASS_NAME)) + return classNameParser.parse(className, AttachmentPoint.NONE); return new DefaultCustomType(className); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java index 05ae3980823..d6afbe0380a 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java @@ -28,6 +28,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; /** * Manages the two-way conversion between a CQL type and a Java type. @@ -234,4 +235,9 @@ default boolean accepts(@NonNull DataType cqlType) { */ @Nullable JavaTypeT parse(@Nullable String value); + + @NonNull + default Optional serializedSize() { + return Optional.empty(); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java index 9f2fd5cc69e..68f1b07b106 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java @@ -210,13 +210,13 @@ public static TypeCodec tupleOf(@NonNull TupleType cqlType) { return new TupleCodec(cqlType); } - public static TypeCodec> vectorOf( + public static TypeCodec> vectorOf( @NonNull VectorType type, @NonNull TypeCodec subtypeCodec) { return new VectorCodec( DataTypes.vectorOf(subtypeCodec.getCqlType(), type.getDimensions()), subtypeCodec); } - public static TypeCodec> vectorOf( + public static TypeCodec> vectorOf( int dimensions, @NonNull TypeCodec subtypeCodec) { return new VectorCodec(DataTypes.vectorOf(subtypeCodec.getCqlType(), dimensions), subtypeCodec); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java index c6482d4f4d1..d22b6f1bfaf 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java @@ -151,8 +151,7 @@ public static GenericType> setOf(@NonNull GenericType elementType) } @NonNull - public static GenericType> vectorOf( - @NonNull Class elementType) { + public static GenericType> vectorOf(@NonNull Class elementType) { TypeToken> token = new TypeToken>() {}.where( new TypeParameter() {}, TypeToken.of(elementType)); @@ -160,8 +159,7 @@ public static GenericType> vectorOf( } @NonNull - public static GenericType> vectorOf( - @NonNull GenericType elementType) { + public static GenericType> vectorOf(@NonNull GenericType elementType) { TypeToken> token = new TypeToken>() {}.where(new TypeParameter() {}, elementType.token); return new GenericType<>(token); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java index fd6f1a4bd51..bf252d0bc57 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java @@ -34,6 +34,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -164,6 +165,13 @@ private DataType parse( return new DefaultTupleType(componentTypesBuilder.build(), attachmentPoint); } + if (next.startsWith("org.apache.cassandra.db.marshal.VectorType")) { + Iterator rawTypes = parser.getTypeParameters().iterator(); + DataType subtype = parse(rawTypes.next(), userTypes, attachmentPoint, logPrefix); + int dimensions = Integer.parseInt(rawTypes.next()); + return DataTypes.vectorOf(subtype, dimensions); + } + DataType type = NATIVE_TYPES_BY_CLASS_NAME.get(next); return type == null ? DataTypes.custom(toParse) : type; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java index c9180d44edc..0b1ced94769 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java @@ -78,7 +78,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), subtype, dimensions); + return Objects.hash(DefaultVectorType.class, subtype, dimensions); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java index 2b3b8255cc1..8496da17fa6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Long parse(@Nullable String value) { String.format("Cannot parse 64-bits long value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java index 7a982a9e6ca..af388982be9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -98,4 +99,10 @@ public Boolean parse(@Nullable String value) { String.format("Cannot parse boolean value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(1); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java index 28eff6f9463..b01847517d9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Double parse(@Nullable String value) { String.format("Cannot parse 64-bits double value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java index 11786dbc77d..fd851edfad3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Float parse(@Nullable String value) { String.format("Cannot parse 32-bits float value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(4); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java index e5bb530ba79..b11b164a445 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Integer parse(@Nullable String value) { String.format("Cannot parse 32-bits int value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(4); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java index eeba3c7c66c..964f774c8d9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java @@ -33,6 +33,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.Date; +import java.util.Optional; import java.util.TimeZone; import net.jcip.annotations.ThreadSafe; @@ -293,4 +294,10 @@ public Instant parse(@Nullable String value) { String.format("Cannot parse timestamp value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java index 57feac4ae7e..cc5f48dbe52 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.UUID; import net.jcip.annotations.ThreadSafe; @@ -95,4 +96,10 @@ public UUID parse(@Nullable String value) { String.format("Cannot parse UUID value from \"%s\"", value), e); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(16); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java index 2c4d2200b13..1f8ce1a7166 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java @@ -24,7 +24,7 @@ import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.reflect.GenericType; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; -import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; +import com.datastax.oss.driver.internal.core.type.util.VIntCoding; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; @@ -32,8 +32,10 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.stream.Collectors; -public class VectorCodec implements TypeCodec> { +public class VectorCodec implements TypeCodec> { private final VectorType cqlType; private final GenericType> javaType; @@ -55,6 +57,14 @@ public GenericType> getJavaType() { return this.javaType; } + @NonNull + @Override + public Optional serializedSize() { + return subtypeCodec.serializedSize().isPresent() + ? Optional.of(subtypeCodec.serializedSize().get() * cqlType.getDimensions()) + : Optional.empty(); + } + @NonNull @Override public DataType getCqlType() { @@ -65,6 +75,7 @@ public DataType getCqlType() { @Override public ByteBuffer encode( @Nullable CqlVector value, @NonNull ProtocolVersion protocolVersion) { + boolean isVarSized = !subtypeCodec.serializedSize().isPresent(); if (value == null || cqlType.getDimensions() <= 0) { return null; } @@ -92,14 +103,28 @@ public ByteBuffer encode( if (valueBuff == null) { throw new NullPointerException("Vector elements cannot encode to CQL NULL"); } - allValueBuffsSize += valueBuff.limit(); + int elementSize = valueBuff.limit(); + if (isVarSized) { + allValueBuffsSize += VIntCoding.computeVIntSize(elementSize); + } + allValueBuffsSize += elementSize; valueBuff.rewind(); valueBuffs[i] = valueBuff; } + // if too many elements, throw + if (values.hasNext()) { + throw new IllegalArgumentException( + String.format( + "Too many elements; must provide elements for %d dimensions", + cqlType.getDimensions())); + } /* Since we already did an early return for <= 0 dimensions above */ assert valueBuffs.length > 0; ByteBuffer rv = ByteBuffer.allocate(allValueBuffsSize); for (int i = 0; i < cqlType.getDimensions(); ++i) { + if (isVarSized) { + VIntCoding.writeUnsignedVInt32(valueBuffs[i].remaining(), rv); + } rv.put(valueBuffs[i]); } rv.flip(); @@ -114,39 +139,58 @@ public CqlVector decode( return null; } - /* Determine element size by dividing count of remaining bytes by number of elements. This should have a remainder - of zero if we assume all elements are of uniform size (which is really a terrible assumption). - - TODO: We should probably tweak serialization format for vectors if we're going to allow them for arbitrary subtypes. - Elements should at least precede themselves with their size (along the lines of what lists do). */ - int elementSize = Math.floorDiv(bytes.remaining(), cqlType.getDimensions()); - if (!(bytes.remaining() % cqlType.getDimensions() == 0)) { - throw new IllegalArgumentException( - String.format( - "Expected elements of uniform size, observed %d elements with total bytes %d", - cqlType.getDimensions(), bytes.remaining())); - } - + // Upfront check for fixed-size types only + subtypeCodec + .serializedSize() + .ifPresent( + (fixed_size) -> { + if (bytes.remaining() != cqlType.getDimensions() * fixed_size) { + throw new IllegalArgumentException( + String.format( + "Expected elements of uniform size, observed %d elements with total bytes %d", + cqlType.getDimensions(), bytes.remaining())); + } + }); + ; ByteBuffer slice = bytes.slice(); List rv = new ArrayList(cqlType.getDimensions()); for (int i = 0; i < cqlType.getDimensions(); ++i) { - // Set the limit for the current element + + int size = + subtypeCodec + .serializedSize() + .orElseGet(() -> VIntCoding.getUnsignedVInt32(slice, slice.position())); + // If we aren't dealing with a fixed-size type we need to move the current slice position + // beyond the vint-encoded size of the current element. Ideally this would be + // serializedSize().ifNotPresent(Consumer) but the Optional API isn't doing us any favors + // there. + if (!subtypeCodec.serializedSize().isPresent()) + slice.position(slice.position() + VIntCoding.computeUnsignedVIntSize(size)); int originalPosition = slice.position(); - slice.limit(originalPosition + elementSize); + slice.limit(originalPosition + size); rv.add(this.subtypeCodec.decode(slice, protocolVersion)); // Move to the start of the next element - slice.position(originalPosition + elementSize); + slice.position(originalPosition + size); // Reset the limit to the end of the buffer slice.limit(slice.capacity()); } + // if too many elements, throw + if (slice.hasRemaining()) { + throw new IllegalArgumentException( + String.format( + "Too many elements; must provide elements for %d dimensions", + cqlType.getDimensions())); + } + return CqlVector.newInstance(rv); } @NonNull @Override - public String format(@Nullable CqlVector value) { - return value == null ? "NULL" : Iterables.toString(value); + public String format(CqlVector value) { + if (value == null) return "NULL"; + return value.stream().map(subtypeCodec::format).collect(Collectors.joining(", ", "[", "]")); } @Nullable diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java index a15495a432d..12e3e839d2a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java @@ -31,6 +31,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.Objects; +import java.util.Optional; import net.jcip.annotations.Immutable; /** @@ -114,4 +115,10 @@ public String format(@Nullable Long value) { Instant instant = value == null ? null : Instant.ofEpochMilli(value); return timestampCodec.format(instant); } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java index 495d6227d93..3af5a30ba27 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java @@ -394,10 +394,9 @@ protected GenericType inspectType(@NonNull Object value, @Nullable DataType c "Can't infer vector codec because the first element is null " + "(note that CQL does not allow null values in collections)"); } - GenericType elementType = - (GenericType) - inspectType( - firstElement, cqlType == null ? null : ((VectorType) cqlType).getElementType()); + GenericType elementType = + inspectType( + firstElement, cqlType == null ? null : ((VectorType) cqlType).getElementType()); return GenericType.vectorOf(elementType); } } else { @@ -421,8 +420,7 @@ protected GenericType inferJavaTypeFromCqlType(@NonNull DataType cqlType) { inferJavaTypeFromCqlType(keyType), inferJavaTypeFromCqlType(valueType)); } else if (cqlType instanceof VectorType) { DataType elementType = ((VectorType) cqlType).getElementType(); - GenericType numberType = - (GenericType) inferJavaTypeFromCqlType(elementType); + GenericType numberType = inferJavaTypeFromCqlType(elementType); return GenericType.vectorOf(numberType); } switch (cqlType.getProtocolCode()) { @@ -657,7 +655,7 @@ protected TypeCodec createCodec( /* For a vector type we'll always get back an instance of TypeCodec due to the * type of CqlVector... but getElementCodecForCqlAndJavaType() is a generalized function that can't * return this more precise type. Thus the cast here. */ - TypeCodec elementCodec = + TypeCodec elementCodec = uncheckedCast(getElementCodecForCqlAndJavaType(vectorType, token, isJavaCovariant)); return TypeCodecs.vectorOf(vectorType, elementCodec); } else if (cqlType instanceof CustomType diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java index 5ee375a81e5..552f84f2ae1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java @@ -49,6 +49,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.ByteBuffer; /** * Variable length encoding inspired from Google > 6; } + + public static void writeUnsignedVInt32(int value, ByteBuffer output) { + writeUnsignedVInt((long) value, output); + } + + public static void writeUnsignedVInt(long value, ByteBuffer output) { + int size = VIntCoding.computeUnsignedVIntSize(value); + if (size == 1) { + output.put((byte) value); + return; + } + + output.put(VIntCoding.encodeVInt(value, size), 0, size); + } + + /** + * Read up to a 32-bit integer back, using the unsigned (no zigzag) encoding. + * + *

Note this method is the same as {@link #readUnsignedVInt(DataInput)}, except that we do + * *not* block if there are not enough bytes in the buffer to reconstruct the value. + * + * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer + */ + public static int getUnsignedVInt32(ByteBuffer input, int readerIndex) { + return checkedCast(getUnsignedVInt(input, readerIndex)); + } + + public static long getUnsignedVInt(ByteBuffer input, int readerIndex) { + return getUnsignedVInt(input, readerIndex, input.limit()); + } + + public static long getUnsignedVInt(ByteBuffer input, int readerIndex, int readerLimit) { + if (readerIndex < 0) + throw new IllegalArgumentException( + "Reader index should be non-negative, but was " + readerIndex); + + if (readerIndex >= readerLimit) return -1; + + int firstByte = input.get(readerIndex++); + + // Bail out early if this is one byte, necessary or it fails later + if (firstByte >= 0) return firstByte; + + int size = numberOfExtraBytesToRead(firstByte); + if (readerIndex + size > readerLimit) return -1; + + long retval = firstByte & firstByteValueMask(size); + for (int ii = 0; ii < size; ii++) { + byte b = input.get(readerIndex++); + retval <<= 8; + retval |= b & 0xff; + } + + return retval; + } + + public static int checkedCast(long value) { + int result = (int) value; + if ((long) result != value) throw new VIntOutOfRangeException(value); + return result; + } + + /** + * Throw when attempting to decode a vint and the output type doesn't have enough space to fit the + * value that was decoded + */ + public static class VIntOutOfRangeException extends RuntimeException { + public final long value; + + private VIntOutOfRangeException(long value) { + super(value + " is out of range for a 32-bit integer"); + this.value = value; + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java index 90f4cc6e776..3e0872cb946 100644 --- a/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java @@ -23,56 +23,60 @@ import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.internal.SerializationHelper; -import com.datastax.oss.driver.shaded.guava.common.collect.Iterators; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.ByteArrayInputStream; import java.io.ObjectInputStream; import java.io.ObjectStreamException; +import java.time.LocalTime; import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.assertj.core.util.Lists; import org.junit.Test; +import org.junit.runner.RunWith; +@RunWith(DataProviderRunner.class) public class CqlVectorTest { - private static final Float[] VECTOR_ARGS = {1.0f, 2.5f}; - - private void validate_built_vector(CqlVector vec) { + @DataProvider + public static Object[][] dataProvider() { + return new Object[][] { + {new Float[] {1.0f, 2.5f}}, + {new LocalTime[] {LocalTime.of(1, 2), LocalTime.of(3, 4)}}, + {new List[] {Arrays.asList(1, 2), Arrays.asList(3, 4)}}, + {new CqlVector[] {CqlVector.newInstance("a", "bc"), CqlVector.newInstance("d", "ef")}} + }; + } + private void validate_built_vector(CqlVector vec, Object[] expectedVals) { assertThat(vec.size()).isEqualTo(2); assertThat(vec.isEmpty()).isFalse(); - assertThat(vec.get(0)).isEqualTo(VECTOR_ARGS[0]); - assertThat(vec.get(1)).isEqualTo(VECTOR_ARGS[1]); + assertThat(vec.get(0)).isEqualTo(expectedVals[0]); + assertThat(vec.get(1)).isEqualTo(expectedVals[1]); } + @UseDataProvider("dataProvider") @Test - public void should_build_vector_from_elements() { - - validate_built_vector(CqlVector.newInstance(VECTOR_ARGS)); + public void should_build_vector_from_elements(Object[] vals) { + validate_built_vector(CqlVector.newInstance(vals), vals); } @Test - public void should_build_vector_from_list() { - - validate_built_vector(CqlVector.newInstance(Lists.newArrayList(VECTOR_ARGS))); - } - - @Test - public void should_build_vector_from_tostring_output() { - - CqlVector vector1 = CqlVector.newInstance(VECTOR_ARGS); - CqlVector vector2 = CqlVector.from(vector1.toString(), TypeCodecs.FLOAT); - assertThat(vector2).isEqualTo(vector1); + @UseDataProvider("dataProvider") + public void should_build_vector_from_list(Object[] vals) { + validate_built_vector(CqlVector.newInstance(Lists.newArrayList(vals)), vals); } @Test public void should_throw_from_null_string() { - assertThatThrownBy( () -> { CqlVector.from(null, TypeCodecs.FLOAT); @@ -116,101 +120,97 @@ public void should_throw_when_building_with_nulls() { @Test public void should_build_empty_vector() { - CqlVector vector = CqlVector.newInstance(); assertThat(vector.isEmpty()).isTrue(); assertThat(vector.size()).isEqualTo(0); } @Test - public void should_behave_mostly_like_a_list() { - - CqlVector vector = CqlVector.newInstance(VECTOR_ARGS); - assertThat(vector.get(0)).isEqualTo(VECTOR_ARGS[0]); - Float newVal = VECTOR_ARGS[0] * 2; - vector.set(0, newVal); - assertThat(vector.get(0)).isEqualTo(newVal); + @UseDataProvider("dataProvider") + public void should_behave_mostly_like_a_list(T[] vals) { + T[] theArray = Arrays.copyOf(vals, vals.length); + CqlVector vector = CqlVector.newInstance(theArray); + assertThat(vector.get(0)).isEqualTo(theArray[0]); + vector.set(0, theArray[1]); + assertThat(vector.get(0)).isEqualTo(theArray[1]); assertThat(vector.isEmpty()).isFalse(); assertThat(vector.size()).isEqualTo(2); - assertThat(Iterators.toArray(vector.iterator(), Float.class)).isEqualTo(VECTOR_ARGS); + Iterator iterator = vector.iterator(); + assertThat(iterator.next()).isEqualTo(theArray[1]); + assertThat(iterator.next()).isEqualTo(theArray[1]); } @Test - public void should_play_nicely_with_streams() { - - CqlVector vector = CqlVector.newInstance(VECTOR_ARGS); - List results = + @UseDataProvider("dataProvider") + public void should_play_nicely_with_streams(T[] vals) { + CqlVector vector = CqlVector.newInstance(vals); + List results = vector.stream() - .map((f) -> f * 2) - .collect(Collectors.toCollection(() -> new ArrayList())); + .map(Object::toString) + .collect(Collectors.toCollection(() -> new ArrayList())); for (int i = 0; i < vector.size(); ++i) { - assertThat(results.get(i)).isEqualTo(vector.get(i) * 2); + assertThat(results.get(i)).isEqualTo(vector.get(i).toString()); } } @Test - public void should_reflect_changes_to_mutable_list() { - - List theList = Lists.newArrayList(1.1f, 2.2f, 3.3f); - CqlVector vector = CqlVector.newInstance(theList); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(3.3f); - - float newVal1 = 4.4f; - theList.set(2, newVal1); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(newVal1); + @UseDataProvider("dataProvider") + public void should_reflect_changes_to_mutable_list(T[] vals) { + List theList = Lists.newArrayList(vals); + CqlVector vector = CqlVector.newInstance(theList); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(vals[1]); - float newVal2 = 5.5f; - theList.add(newVal2); - assertThat(vector.size()).isEqualTo(4); - assertThat(vector.get(3)).isEqualTo(newVal2); + T newVal = vals[0]; + theList.set(1, newVal); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(newVal); } @Test - public void should_reflect_changes_to_array() { - - Float[] theArray = new Float[] {1.1f, 2.2f, 3.3f}; - CqlVector vector = CqlVector.newInstance(theArray); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(3.3f); + @UseDataProvider("dataProvider") + public void should_reflect_changes_to_array(T[] vals) { + T[] theArray = Arrays.copyOf(vals, vals.length); + CqlVector vector = CqlVector.newInstance(theArray); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(theArray[1]); - float newVal1 = 4.4f; - theArray[2] = newVal1; - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(newVal1); + T newVal = theArray[0]; + theArray[1] = newVal; + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(newVal); } @Test - public void should_correctly_compare_vectors() { - - Float[] args = VECTOR_ARGS.clone(); - CqlVector vector1 = CqlVector.newInstance(args); - CqlVector vector2 = CqlVector.newInstance(args); - CqlVector vector3 = CqlVector.newInstance(Lists.newArrayList(args)); + @UseDataProvider("dataProvider") + public void should_correctly_compare_vectors(T[] vals) { + CqlVector vector1 = CqlVector.newInstance(vals); + CqlVector vector2 = CqlVector.newInstance(vals); + CqlVector vector3 = CqlVector.newInstance(Lists.newArrayList(vals)); assertThat(vector1).isNotSameAs(vector2); assertThat(vector1).isEqualTo(vector2); assertThat(vector1).isNotSameAs(vector3); assertThat(vector1).isEqualTo(vector3); - Float[] differentArgs = args.clone(); - float newVal = differentArgs[0] * 2; + T[] differentArgs = Arrays.copyOf(vals, vals.length); + T newVal = differentArgs[1]; differentArgs[0] = newVal; - CqlVector vector4 = CqlVector.newInstance(differentArgs); + CqlVector vector4 = CqlVector.newInstance(differentArgs); assertThat(vector1).isNotSameAs(vector4); assertThat(vector1).isNotEqualTo(vector4); - Float[] biggerArgs = Arrays.copyOf(args, args.length + 1); + T[] biggerArgs = Arrays.copyOf(vals, vals.length + 1); biggerArgs[biggerArgs.length - 1] = newVal; - CqlVector vector5 = CqlVector.newInstance(biggerArgs); + CqlVector vector5 = CqlVector.newInstance(biggerArgs); assertThat(vector1).isNotSameAs(vector5); assertThat(vector1).isNotEqualTo(vector5); } @Test - public void should_serialize_and_deserialize() throws Exception { - CqlVector initial = CqlVector.newInstance(VECTOR_ARGS); - CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); + @UseDataProvider("dataProvider") + public void should_serialize_and_deserialize(T[] vals) throws Exception { + CqlVector initial = CqlVector.newInstance(vals); + CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); assertThat(deserialized).isEqualTo(initial); } @@ -222,21 +222,22 @@ public void should_serialize_and_deserialize_empty_vector() throws Exception { } @Test - public void should_serialize_and_deserialize_unserializable_list() throws Exception { - CqlVector initial = + @UseDataProvider("dataProvider") + public void should_serialize_and_deserialize_unserializable_list(T[] vals) throws Exception { + CqlVector initial = CqlVector.newInstance( - new AbstractList() { + new AbstractList() { @Override - public Float get(int index) { - return VECTOR_ARGS[index]; + public T get(int index) { + return vals[index]; } @Override public int size() { - return VECTOR_ARGS.length; + return vals.length; } }); - CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); + CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); assertThat(deserialized).isEqualTo(initial); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java index 969d35cbbbe..17c78514127 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java @@ -20,122 +20,255 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.data.CqlVector; +import com.datastax.oss.driver.api.core.type.DataType; import com.datastax.oss.driver.api.core.type.DataTypes; -import com.datastax.oss.driver.api.core.type.VectorType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; -import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; -import java.util.Arrays; +import com.datastax.oss.protocol.internal.util.Bytes; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.nio.ByteBuffer; +import java.time.LocalTime; +import java.util.HashMap; +import org.apache.commons.lang3.ArrayUtils; import org.junit.Test; +import org.junit.runner.RunWith; -public class VectorCodecTest extends CodecTestBase> { +@RunWith(DataProviderRunner.class) +public class VectorCodecTest { - private static final Float[] VECTOR_ARGS = {1.0f, 2.5f}; - - private static final CqlVector VECTOR = CqlVector.newInstance(VECTOR_ARGS); - - private static final String VECTOR_HEX_STRING = "0x" + "3f800000" + "40200000"; - - private static final String FORMATTED_VECTOR = "[1.0, 2.5]"; - - public VectorCodecTest() { - VectorType vectorType = DataTypes.vectorOf(DataTypes.FLOAT, 2); - this.codec = TypeCodecs.vectorOf(vectorType, TypeCodecs.FLOAT); + @DataProvider + public static Object[] dataProvider() { + HashMap map1 = new HashMap<>(); + map1.put(1, "a"); + HashMap map2 = new HashMap<>(); + map2.put(2, "b"); + return new TestDataContainer[] { + new TestDataContainer( + DataTypes.FLOAT, + new Float[] {1.0f, 2.5f}, + "[1.0, 2.5]", + Bytes.fromHexString("0x3f80000040200000")), + new TestDataContainer( + DataTypes.ASCII, + new String[] {"ab", "cde"}, + "['ab', 'cde']", + Bytes.fromHexString("0x02616203636465")), + new TestDataContainer( + DataTypes.BIGINT, + new Long[] {1L, 2L}, + "[1, 2]", + Bytes.fromHexString("0x00000000000000010000000000000002")), + new TestDataContainer( + DataTypes.BLOB, + new ByteBuffer[] {Bytes.fromHexString("0xCAFE"), Bytes.fromHexString("0xABCD")}, + "[0xcafe, 0xabcd]", + Bytes.fromHexString("0x02cafe02abcd")), + new TestDataContainer( + DataTypes.BOOLEAN, + new Boolean[] {true, false}, + "[true, false]", + Bytes.fromHexString("0x0100")), + new TestDataContainer( + DataTypes.TIME, + new LocalTime[] {LocalTime.ofNanoOfDay(1), LocalTime.ofNanoOfDay(2)}, + "['00:00:00.000000001', '00:00:00.000000002']", + Bytes.fromHexString("0x080000000000000001080000000000000002")), + new TestDataContainer( + DataTypes.mapOf(DataTypes.INT, DataTypes.ASCII), + new HashMap[] {map1, map2}, + "[{1:'a'}, {2:'b'}]", + Bytes.fromHexString( + "0x110000000100000004000000010000000161110000000100000004000000020000000162")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.INT, 1), + new CqlVector[] {CqlVector.newInstance(1), CqlVector.newInstance(2)}, + "[[1], [2]]", + Bytes.fromHexString("0x0000000100000002")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.TEXT, 1), + new CqlVector[] {CqlVector.newInstance("ab"), CqlVector.newInstance("cdef")}, + "[['ab'], ['cdef']]", + Bytes.fromHexString("0x03026162050463646566")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.vectorOf(DataTypes.FLOAT, 2), 1), + new CqlVector[] { + CqlVector.newInstance(CqlVector.newInstance(1.0f, 2.5f)), + CqlVector.newInstance(CqlVector.newInstance(3.0f, 4.5f)) + }, + "[[[1.0, 2.5]], [[3.0, 4.5]]]", + Bytes.fromHexString("0x3f800000402000004040000040900000")) + }; } + @UseDataProvider("dataProvider") @Test - public void should_encode() { - assertThat(encode(VECTOR)).isEqualTo(VECTOR_HEX_STRING); - assertThat(encode(null)).isNull(); + public void should_encode(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.encode(vector, ProtocolVersion.DEFAULT)).isEqualTo(testData.getBytes()); } - /** Too few eleements will cause an exception, extra elements will be silently ignored */ @Test - public void should_throw_on_encode_with_too_few_elements() { - assertThatThrownBy(() -> encode(VECTOR.subVector(0, 1))) + @UseDataProvider("dataProvider") + public void should_throw_on_encode_with_too_few_elements(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy( + () -> + codec.encode( + CqlVector.newInstance(testData.getValues()[0]), ProtocolVersion.DEFAULT)) .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_throw_on_encode_with_empty_list() { - assertThatThrownBy(() -> encode(CqlVector.newInstance())) + @UseDataProvider("dataProvider") + public void should_throw_on_encode_with_too_many_elements(TestDataContainer testData) { + Object[] doubled = ArrayUtils.addAll(testData.getValues(), testData.getValues()); + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy(() -> codec.encode(CqlVector.newInstance(doubled), ProtocolVersion.DEFAULT)) .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_encode_with_too_many_elements() { - Float[] doubledVectorContents = Arrays.copyOf(VECTOR_ARGS, VECTOR_ARGS.length * 2); - System.arraycopy(VECTOR_ARGS, 0, doubledVectorContents, VECTOR_ARGS.length, VECTOR_ARGS.length); - assertThat(encode(CqlVector.newInstance(doubledVectorContents))).isEqualTo(VECTOR_HEX_STRING); + @UseDataProvider("dataProvider") + public void should_decode(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.decode(testData.getBytes(), ProtocolVersion.DEFAULT)) + .isEqualTo(CqlVector.newInstance(testData.getValues())); } @Test - public void should_decode() { - assertThat(decode(VECTOR_HEX_STRING)).isEqualTo(VECTOR); - assertThat(decode("0x")).isNull(); - assertThat(decode(null)).isNull(); + @UseDataProvider("dataProvider") + public void should_throw_on_decode_if_too_few_bytes(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + int lastIndex = testData.getBytes().remaining() - 1; + assertThatThrownBy( + () -> + codec.decode( + (ByteBuffer) testData.getBytes().duplicate().limit(lastIndex), + ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_throw_on_decode_if_too_few_bytes() { - // Dropping 4 bytes would knock off exactly 1 float, anything less than that would be something - // we couldn't parse a float out of - for (int i = 1; i <= 3; ++i) { - // 2 chars of hex encoded string = 1 byte - int lastIndex = VECTOR_HEX_STRING.length() - (2 * i); - assertThatThrownBy(() -> decode(VECTOR_HEX_STRING.substring(0, lastIndex))) - .isInstanceOf(IllegalArgumentException.class); - } + @UseDataProvider("dataProvider") + public void should_throw_on_decode_if_too_many_bytes(TestDataContainer testData) { + ByteBuffer doubled = ByteBuffer.allocate(testData.getBytes().remaining() * 2); + doubled.put(testData.getBytes().duplicate()).put(testData.getBytes().duplicate()).flip(); + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy(() -> codec.decode(doubled, ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_format() { - assertThat(format(VECTOR)).isEqualTo(FORMATTED_VECTOR); - assertThat(format(null)).isEqualTo("NULL"); + @UseDataProvider("dataProvider") + public void should_format(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.format(vector)).isEqualTo(testData.getFormatted()); } @Test - public void should_parse() { - assertThat(parse(FORMATTED_VECTOR)).isEqualTo(VECTOR); - assertThat(parse("NULL")).isNull(); - assertThat(parse("null")).isNull(); - assertThat(parse("")).isNull(); - assertThat(parse(null)).isNull(); + @UseDataProvider("dataProvider") + public void should_parse(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.parse(testData.getFormatted())) + .isEqualTo(CqlVector.newInstance(testData.getValues())); } @Test - public void should_accept_data_type() { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 2))).isTrue(); - assertThat(codec.accepts(DataTypes.INT)).isFalse(); + @UseDataProvider("dataProvider") + public void should_accept_data_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 2))).isTrue(); + assertThat(codec.accepts(new DefaultVectorType(DataTypes.custom("non-existent"), 2))).isFalse(); } @Test - public void should_accept_vector_type_correct_dimension_only() { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 0))).isFalse(); - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 1))).isFalse(); - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 2))).isTrue(); - for (int i = 3; i < 1000; ++i) { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, i))).isFalse(); - } + @UseDataProvider("dataProvider") + public void should_accept_vector_type_correct_dimension_only(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 0))).isFalse(); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 1))).isFalse(); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 3))).isFalse(); } @Test - public void should_accept_generic_type() { - assertThat(codec.accepts(GenericType.vectorOf(GenericType.FLOAT))).isTrue(); - assertThat(codec.accepts(GenericType.vectorOf(GenericType.INTEGER))).isFalse(); - assertThat(codec.accepts(GenericType.of(Integer.class))).isFalse(); + @UseDataProvider("dataProvider") + public void should_accept_generic_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(codec.getJavaType())).isTrue(); } @Test - public void should_accept_raw_type() { + @UseDataProvider("dataProvider") + public void should_accept_raw_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); assertThat(codec.accepts(CqlVector.class)).isTrue(); assertThat(codec.accepts(Integer.class)).isFalse(); } @Test - public void should_accept_object() { - assertThat(codec.accepts(VECTOR)).isTrue(); + @UseDataProvider("dataProvider") + public void should_accept_object(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.accepts(vector)).isTrue(); assertThat(codec.accepts(Integer.MIN_VALUE)).isFalse(); } + + @Test + public void should_handle_null_and_empty() { + TypeCodec> codec = getCodec(DataTypes.FLOAT); + assertThat(codec.encode(null, ProtocolVersion.DEFAULT)).isNull(); + assertThat(codec.decode(Bytes.fromHexString("0x"), ProtocolVersion.DEFAULT)).isNull(); + assertThat(codec.format(null)).isEqualTo("NULL"); + assertThat(codec.parse("NULL")).isNull(); + assertThat(codec.parse("null")).isNull(); + assertThat(codec.parse("")).isNull(); + assertThat(codec.parse(null)).isNull(); + assertThatThrownBy(() -> codec.encode(CqlVector.newInstance(), ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static TypeCodec> getCodec(DataType dataType) { + return TypeCodecs.vectorOf( + DataTypes.vectorOf(dataType, 2), CodecRegistry.DEFAULT.codecFor(dataType)); + } + + private static class TestDataContainer { + private final DataType dataType; + private final Object[] values; + private final String formatted; + private final ByteBuffer bytes; + + public TestDataContainer( + DataType dataType, Object[] values, String formatted, ByteBuffer bytes) { + this.dataType = dataType; + this.values = values; + this.formatted = formatted; + this.bytes = bytes; + } + + public DataType getDataType() { + return dataType; + } + + public Object[] getValues() { + return values; + } + + public String getFormatted() { + return formatted; + } + + public ByteBuffer getBytes() { + return bytes; + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java index 1f3f6bbff97..4c0298bafad 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java @@ -337,6 +337,26 @@ public static Object[][] collectionsWithCqlAndJavaTypes() GenericType.vectorOf(BigInteger.class), CqlVector.newInstance(BigInteger.ONE) }, + // vector with arbitrary types + { + DataTypes.vectorOf(DataTypes.TEXT, 2), + GenericType.vectorOf(String.class), + GenericType.vectorOf(String.class), + CqlVector.newInstance("abc", "de") + }, + { + DataTypes.vectorOf(DataTypes.TIME, 2), + GenericType.vectorOf(LocalTime.class), + GenericType.vectorOf(LocalTime.class), + CqlVector.newInstance(LocalTime.MIDNIGHT, LocalTime.NOON) + }, + { + DataTypes.vectorOf(DataTypes.vectorOf(DataTypes.TINYINT, 2), 2), + GenericType.vectorOf(GenericType.vectorOf(Byte.class)), + GenericType.vectorOf(GenericType.vectorOf(Byte.class)), + CqlVector.newInstance( + CqlVector.newInstance((byte) 1, (byte) 2), CqlVector.newInstance((byte) 3, (byte) 4)) + }, }; } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java new file mode 100644 index 00000000000..b85d6d66844 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.type.util; + +import static org.junit.Assert.assertEquals; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.nio.ByteBuffer; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(DataProviderRunner.class) +public class VIntCodingTest { + @DataProvider + public static Object[] roundTripTestValues() { + return new Integer[] { + Integer.MAX_VALUE + 1, + Integer.MAX_VALUE, + Integer.MAX_VALUE - 1, + Integer.MIN_VALUE, + Integer.MIN_VALUE + 1, + Integer.MIN_VALUE - 1, + 0, + -1, + 1 + }; + }; + + private static final long[] LONGS = + new long[] { + 53L, + 10201L, + 1097151L, + 168435455L, + 33251130335L, + 3281283447775L, + 417672546086779L, + 52057592037927932L, + 72057594037927937L + }; + + @Test + public void should_compute_unsigned_vint_size() { + for (int i = 0; i < LONGS.length; i++) { + long val = LONGS[i]; + assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val)); + } + } + + @Test + @UseDataProvider("roundTripTestValues") + public void should_write_and_read_unsigned_vint_32(int value) { + ByteBuffer bb = ByteBuffer.allocate(9); + + VIntCoding.writeUnsignedVInt32(value, bb); + bb.flip(); + assertEquals(value, VIntCoding.getUnsignedVInt32(bb, 0)); + } + + @Test + @UseDataProvider("roundTripTestValues") + public void should_write_and_read_unsigned_vint(int value) { + ByteBuffer bb = ByteBuffer.allocate(9); + + VIntCoding.writeUnsignedVInt(value, bb); + bb.flip(); + assertEquals(value, VIntCoding.getUnsignedVInt(bb, 0)); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java index a33c8704876..e3d891454de 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java @@ -32,6 +32,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.core.data.CqlDuration; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.data.SettableByIndex; import com.datastax.oss.driver.api.core.data.SettableByName; import com.datastax.oss.driver.api.core.data.TupleValue; @@ -183,6 +184,7 @@ public static Object[][] typeSamples() { // 5) include map // 6) include tuple // 7) include udt + // 8) include vector return Arrays.stream(primitiveSamples) .flatMap( o -> { @@ -263,6 +265,30 @@ public static Object[][] typeSamples() { UdtValue udtValue2 = udt.newValue(1, o[1]); samples.add(new Object[] {udt, udtValue2}); + if (CCM_RULE.getCassandraVersion().compareTo(Version.parse("5.0")) >= 0) { + // vector of type + CqlVector vector = CqlVector.newInstance(o[1]); + samples.add(new Object[] {DataTypes.vectorOf(dataType, 1), vector}); + } + + return samples.stream(); + }) + .toArray(Object[][]::new); + } + + @DataProvider + public static Object[][] addVectors() { + Object[][] previousSamples = typeSamples(); + if (CCM_RULE.getCassandraVersion().compareTo(Version.parse("5.0")) < 0) return previousSamples; + return Arrays.stream(previousSamples) + .flatMap( + o -> { + List samples = new ArrayList<>(); + samples.add(o); + if (o[1] == null) return samples.stream(); + DataType dataType = (DataType) o[0]; + CqlVector vector = CqlVector.newInstance(o[1]); + samples.add(new Object[] {DataTypes.vectorOf(dataType, 1), vector}); return samples.stream(); }) .toArray(Object[][]::new); @@ -278,7 +304,7 @@ public static void createTable() { List columnData = new ArrayList<>(); - for (Object[] sample : typeSamples()) { + for (Object[] sample : addVectors()) { DataType dataType = (DataType) sample[0]; if (!typeToColumnName.containsKey(dataType)) { @@ -308,7 +334,7 @@ private static int nextKey() { return keyCounter.incrementAndGet(); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_using_format( DataType dataType, K value, K expectedPrimitiveValue) { @@ -335,7 +361,7 @@ public void should_insert_non_primary_key_column_simple_statement_using_form readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_positional_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -358,7 +384,7 @@ public void should_insert_non_primary_key_column_simple_statement_positional readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_named_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -382,7 +408,7 @@ public void should_insert_non_primary_key_column_simple_statement_named_valu readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_bound_statement_positional_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -411,7 +437,7 @@ public void should_insert_non_primary_key_column_bound_statement_positional_ readValue(boundSelect, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_bound_statement_named_value( DataType dataType, K value, K expectedPrimitiveValue) { From 04d34a8989b89c1addaab7f248b1fa9aa535da5e Mon Sep 17 00:00:00 2001 From: Alex Sasnouskikh Date: Thu, 30 Jan 2025 22:36:29 +0000 Subject: [PATCH 36/77] JAVA-3168 Copy node info for contact points on initial node refresh only from first match by endpoint patch by Alex Sasnouskikh; reviewed by Andy Tolbert and Alexandre Dura for JAVA-3168 --- .../core/metadata/InitialNodeListRefresh.java | 36 +++++++++++-------- .../metadata/InitialNodeListRefreshTest.java | 34 ++++++++++++++++-- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java index c21d5d8171e..517bfca27fa 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java @@ -18,14 +18,16 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metadata.token.TokenFactory; import com.datastax.oss.driver.internal.core.metadata.token.TokenFactoryRegistry; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -63,22 +65,31 @@ public Result compute( TokenFactory tokenFactory = null; Map newNodes = new HashMap<>(); + // Contact point nodes don't have host ID as well as other info yet, so we fill them with node + // info found on first match by endpoint + Set matchedContactPoints = new HashSet<>(); + List addedNodes = new ArrayList<>(); for (NodeInfo nodeInfo : nodeInfos) { UUID hostId = nodeInfo.getHostId(); if (newNodes.containsKey(hostId)) { LOG.warn( "[{}] Found duplicate entries with host_id {} in system.peers, " - + "keeping only the first one", + + "keeping only the first one {}", logPrefix, - hostId); + hostId, + newNodes.get(hostId)); } else { EndPoint endPoint = nodeInfo.getEndPoint(); - DefaultNode node = findIn(contactPoints, endPoint); - if (node == null) { + DefaultNode contactPointNode = findContactPointNode(endPoint); + DefaultNode node; + if (contactPointNode == null || matchedContactPoints.contains(endPoint)) { node = new DefaultNode(endPoint, context); + addedNodes.add(node); LOG.debug("[{}] Adding new node {}", logPrefix, node); } else { + matchedContactPoints.add(contactPointNode.getEndPoint()); + node = contactPointNode; LOG.debug("[{}] Copying contact point {}", logPrefix, node); } if (tokenMapEnabled && tokenFactory == null && nodeInfo.getPartitioner() != null) { @@ -90,14 +101,11 @@ public Result compute( } ImmutableList.Builder eventsBuilder = ImmutableList.builder(); - - for (DefaultNode newNode : newNodes.values()) { - if (findIn(contactPoints, newNode.getEndPoint()) == null) { - eventsBuilder.add(NodeStateEvent.added(newNode)); - } + for (DefaultNode addedNode : addedNodes) { + eventsBuilder.add(NodeStateEvent.added(addedNode)); } for (DefaultNode contactPoint : contactPoints) { - if (findIn(newNodes.values(), contactPoint.getEndPoint()) == null) { + if (!matchedContactPoints.contains(contactPoint.getEndPoint())) { eventsBuilder.add(NodeStateEvent.removed(contactPoint)); } } @@ -108,10 +116,10 @@ public Result compute( eventsBuilder.build()); } - private DefaultNode findIn(Iterable nodes, EndPoint endPoint) { - for (Node node : nodes) { + private DefaultNode findContactPointNode(EndPoint endPoint) { + for (DefaultNode node : contactPoints) { if (node.getEndPoint().equals(endPoint)) { - return (DefaultNode) node; + return node; } } return null; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java index 095662257f6..3787bf8fe10 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java @@ -48,6 +48,8 @@ public class InitialNodeListRefreshTest { private UUID hostId1; private UUID hostId2; private UUID hostId3; + private UUID hostId4; + private UUID hostId5; @Before public void setup() { @@ -61,10 +63,12 @@ public void setup() { hostId1 = UUID.randomUUID(); hostId2 = UUID.randomUUID(); hostId3 = UUID.randomUUID(); + hostId4 = UUID.randomUUID(); + hostId5 = UUID.randomUUID(); } @Test - public void should_copy_contact_points() { + public void should_copy_contact_points_on_first_endpoint_match_only() { // Given Iterable newInfos = ImmutableList.of( @@ -76,6 +80,17 @@ public void should_copy_contact_points() { DefaultNodeInfo.builder() .withEndPoint(contactPoint2.getEndPoint()) .withHostId(hostId2) + .build(), + DefaultNodeInfo.builder().withEndPoint(endPoint3).withHostId(hostId3).build(), + DefaultNodeInfo.builder() + // address translator can translate node addresses to the same endpoints + .withEndPoint(contactPoint2.getEndPoint()) + .withHostId(hostId4) + .build(), + DefaultNodeInfo.builder() + // address translator can translate node addresses to the same endpoints + .withEndPoint(endPoint3) + .withHostId(hostId5) .build()); InitialNodeListRefresh refresh = new InitialNodeListRefresh(newInfos, ImmutableSet.of(contactPoint1, contactPoint2)); @@ -86,11 +101,26 @@ public void should_copy_contact_points() { // Then // contact points have been copied to the metadata, and completed with missing information Map newNodes = result.newMetadata.getNodes(); - assertThat(newNodes).containsOnlyKeys(hostId1, hostId2); + assertThat(newNodes).containsOnlyKeys(hostId1, hostId2, hostId3, hostId4, hostId5); assertThat(newNodes.get(hostId1)).isEqualTo(contactPoint1); assertThat(contactPoint1.getHostId()).isEqualTo(hostId1); assertThat(newNodes.get(hostId2)).isEqualTo(contactPoint2); assertThat(contactPoint2.getHostId()).isEqualTo(hostId2); + // And + // node has been added for the new endpoint + assertThat(newNodes.get(hostId3).getEndPoint()).isEqualTo(endPoint3); + assertThat(newNodes.get(hostId3).getHostId()).isEqualTo(hostId3); + // And + // nodes have been added for duplicated endpoints + assertThat(newNodes.get(hostId4).getEndPoint()).isEqualTo(contactPoint2.getEndPoint()); + assertThat(newNodes.get(hostId4).getHostId()).isEqualTo(hostId4); + assertThat(newNodes.get(hostId5).getEndPoint()).isEqualTo(endPoint3); + assertThat(newNodes.get(hostId5).getHostId()).isEqualTo(hostId5); + assertThat(result.events) + .containsExactlyInAnyOrder( + NodeStateEvent.added((DefaultNode) newNodes.get(hostId3)), + NodeStateEvent.added((DefaultNode) newNodes.get(hostId4)), + NodeStateEvent.added((DefaultNode) newNodes.get(hostId5))); } @Test From eac7b24d60d87e912ea70ba3ddee2755a6fa28cf Mon Sep 17 00:00:00 2001 From: Luc Boutier Date: Fri, 3 Nov 2023 10:13:13 +0100 Subject: [PATCH 37/77] JAVA-3055: Prevent PreparedStatement cache to be polluted if a request is cancelled. There was a critical issue when the external code cancels a request, indeed the cached CompletableFuture will then always throw a CancellationException. This may happens, for example, when used by reactive like Mono.zip or Mono.firstWithValue. patch by Luc Boutier; reviewed by Alexandre Dutra and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1757 --- .../driver/internal/core/cql/CqlPrepareAsyncProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index ffbc8ee046a..d777e35e50f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -159,7 +159,9 @@ public CompletionStage process( }); } } - return result; + // Return a defensive copy. So if a client cancels its request, the cache won't be impacted + // nor a potential concurrent request. + return result.thenApply(x -> x); // copy() is available only since Java 9 } catch (ExecutionException e) { return CompletableFutures.failedFuture(e.getCause()); } From 64b3568dfa6dd58e8c16d28cafbe8dbf16f1e622 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 3 Feb 2025 17:56:54 -0600 Subject: [PATCH 38/77] Expose a decorator for CqlPrepareAsyncProcessor cache rather than the ability to specify an arbitrary cache from scratch. Also bringing tests from https://github.com/apache/cassandra-java-driver/pull/2003 forward with a few minor changes due to this implementation patch by Bret McGuire; reviewed by Bret McGuire and Andy Tolbert reference: #2008 --- .../core/cql/CqlPrepareAsyncProcessor.java | 11 +- .../core/cql/PreparedStatementCachingIT.java | 13 +- .../cql/PreparedStatementCancellationIT.java | 166 ++++++++++++++++++ 3 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index d777e35e50f..918d75e9ecb 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; import com.datastax.oss.protocol.internal.ProtocolConstants; +import com.google.common.base.Functions; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; import java.util.Map; @@ -45,6 +46,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; +import java.util.function.Function; import net.jcip.annotations.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,14 +64,15 @@ public CqlPrepareAsyncProcessor() { } public CqlPrepareAsyncProcessor(@NonNull Optional context) { - this(CacheBuilder.newBuilder().weakValues().build(), context); + this(context, Functions.identity()); } protected CqlPrepareAsyncProcessor( - Cache> cache, - Optional context) { + Optional context, + Function, CacheBuilder> decorator) { - this.cache = cache; + CacheBuilder baseCache = CacheBuilder.newBuilder().weakValues(); + this.cache = decorator.apply(baseCache).build(); context.ifPresent( (ctx) -> { LOG.info("Adding handler to invalidate cached prepared statements on type changes"); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java index 05ac3bd0e92..617d489fb95 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java @@ -24,7 +24,6 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; import com.datastax.oss.driver.api.core.context.DriverContext; -import com.datastax.oss.driver.api.core.cql.PrepareRequest; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; @@ -41,7 +40,6 @@ import com.datastax.oss.driver.internal.core.session.BuiltInRequestProcessors; import com.datastax.oss.driver.internal.core.session.RequestProcessor; import com.datastax.oss.driver.internal.core.session.RequestProcessorRegistry; -import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.cache.RemovalListener; import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import com.google.common.collect.ImmutableList; @@ -119,11 +117,12 @@ private static class TestCqlPrepareAsyncProcessor extends CqlPrepareAsyncProcess private static final Logger LOG = LoggerFactory.getLogger(PreparedStatementCachingIT.TestCqlPrepareAsyncProcessor.class); - private static RemovalListener> - buildCacheRemoveCallback(@NonNull Optional context) { + private static RemovalListener buildCacheRemoveCallback( + @NonNull Optional context) { return (evt) -> { try { - CompletableFuture future = evt.getValue(); + CompletableFuture future = + (CompletableFuture) evt.getValue(); ByteBuffer queryId = Uninterruptibles.getUninterruptibly(future).getId(); context.ifPresent( ctx -> ctx.getEventBus().fire(new PreparedStatementRemovalEvent(queryId))); @@ -136,9 +135,7 @@ private static class TestCqlPrepareAsyncProcessor extends CqlPrepareAsyncProcess public TestCqlPrepareAsyncProcessor(@NonNull Optional context) { // Default CqlPrepareAsyncProcessor uses weak values here as well. We avoid doing so // to prevent cache entries from unexpectedly disappearing mid-test. - super( - CacheBuilder.newBuilder().removalListener(buildCacheRemoveCallback(context)).build(), - context); + super(context, builder -> builder.removalListener(buildCacheRemoveCallback(context))); } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java new file mode 100644 index 00000000000..d7e581e4606 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.core.cql; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.PrepareRequest; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.context.DefaultDriverContext; +import com.datastax.oss.driver.internal.core.cql.CqlPrepareAsyncProcessor; +import com.datastax.oss.driver.shaded.guava.common.base.Predicates; +import com.datastax.oss.driver.shaded.guava.common.cache.Cache; +import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; +import java.util.concurrent.CompletableFuture; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@Category(IsolatedTests.class) +public class PreparedStatementCancellationIT { + + private CustomCcmRule ccmRule = CustomCcmRule.builder().build(); + + private SessionRule sessionRule = SessionRule.builder(ccmRule).build(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + + @Before + public void setup() { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + session.execute("DROP TABLE IF EXISTS test_table_1"); + session.execute("CREATE TABLE test_table_1 (k int primary key, v int)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (1, 100)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (2, 200)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (3, 300)"); + session.close(); + } + + @After + public void teardown() { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + session.execute("DROP TABLE test_table_1"); + session.close(); + } + + private CompletableFuture toCompletableFuture(CqlSession session, String cql) { + + return session.prepareAsync(cql).toCompletableFuture(); + } + + private CqlPrepareAsyncProcessor findProcessor(CqlSession session) { + + DefaultDriverContext context = (DefaultDriverContext) session.getContext(); + return (CqlPrepareAsyncProcessor) + Iterables.find( + context.getRequestProcessorRegistry().getProcessors(), + Predicates.instanceOf(CqlPrepareAsyncProcessor.class)); + } + + @Test + public void should_cache_valid_cql() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + // Make multiple CompletableFuture requests for the specified CQL, then wait until + // the cached request finishes and confirm that all futures got the same values + String cql = "select v from test_table_1 where k = ?"; + CompletableFuture cf1 = toCompletableFuture(session, cql); + CompletableFuture cf2 = toCompletableFuture(session, cql); + assertThat(cache.size()).isEqualTo(1); + + CompletableFuture future = Iterables.get(cache.asMap().values(), 0); + PreparedStatement stmt = future.get(); + + assertThat(cf1.isDone()).isTrue(); + assertThat(cf2.isDone()).isTrue(); + + assertThat(cf1.join()).isEqualTo(stmt); + assertThat(cf2.join()).isEqualTo(stmt); + } + + // A holdover from work done on JAVA-3055. This probably isn't _desired_ behaviour but this test + // documents the fact that the current driver impl will behave in this way. We should probably + // consider changing this in a future release, although it's worthwhile fully considering the + // implications of such a change. + @Test + public void will_cache_invalid_cql() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + // Verify that we get the CompletableFuture even if the CQL is invalid but that nothing is + // cached + String cql = "select v fromfrom test_table_1 where k = ?"; + CompletableFuture cf = toCompletableFuture(session, cql); + + // join() here should throw exceptions due to the invalid syntax... for purposes of this test we + // can ignore this + try { + cf.join(); + fail(); + } catch (Exception e) { + } + + assertThat(cache.size()).isEqualTo(1); + } + + @Test + public void should_not_affect_cache_if_returned_futures_are_cancelled() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + String cql = "select v from test_table_1 where k = ?"; + CompletableFuture cf = toCompletableFuture(session, cql); + + assertThat(cf.isCancelled()).isFalse(); + assertThat(cf.cancel(false)).isTrue(); + assertThat(cf.isCancelled()).isTrue(); + assertThat(cf.isCompletedExceptionally()).isTrue(); + + // Confirm that cancelling the CompletableFuture returned to the user does _not_ cancel the + // future used within the cache. CacheEntry very deliberately doesn't maintain a reference + // to it's contained CompletableFuture so we have to get at this by secondary effects. + assertThat(cache.size()).isEqualTo(1); + CompletableFuture future = Iterables.get(cache.asMap().values(), 0); + PreparedStatement rv = future.get(); + assertThat(rv).isNotNull(); + assertThat(rv.getQuery()).isEqualTo(cql); + assertThat(cache.size()).isEqualTo(1); + } +} From 94e73d9f4e95cd74b319495663862ad50c63f589 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 5 Feb 2025 15:33:46 -0600 Subject: [PATCH 39/77] ninja-fix Using shaded Guava classes for import in order to make OSGi class paths happy. Major hat tip to Dmitry Konstantinov for the find here! --- .../oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index 918d75e9ecb..a3d11cff054 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -34,11 +34,11 @@ import com.datastax.oss.driver.internal.core.session.RequestProcessor; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.internal.core.util.concurrent.RunOrSchedule; +import com.datastax.oss.driver.shaded.guava.common.base.Functions; import com.datastax.oss.driver.shaded.guava.common.cache.Cache; import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; import com.datastax.oss.protocol.internal.ProtocolConstants; -import com.google.common.base.Functions; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; import java.util.Map; From 610b91b160948c96c2d61aa34021db3cdac73aa2 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 4 Feb 2025 15:22:14 -0600 Subject: [PATCH 40/77] Changelog updates for 4.19.0 --- changelog/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/changelog/README.md b/changelog/README.md index 83ebb44239f..08634bcb834 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -21,6 +21,30 @@ under the License. +### 4.19.0 + +- [bug] JAVA-3055: Prevent PreparedStatement cache to be polluted if a request is cancelled. +- [bug] JAVA-3168: Copy node info for contact points on initial node refresh only from first match by endpoint +- [improvement] JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0) +- [improvement] CASSJAVA-53: Update Guava version used in cassandra-java-driver +- [improvement] JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder +- [bug] CASSJAVA-55: Remove setting "Host" header for metadata requests +- [bug] JAVA-3057: Allow decoding a UDT that has more fields than expected +- [improvement] CASSJAVA-52: Bring java-driver-shaded-guava into the repo as a submodule +- [bug] CASSJAVA-2: TableMetadata#describe produces invalid CQL when a type of a column is a vector +- [bug] JAVA-3051: Memory leak in DefaultLoadBalancingPolicy measurement of response times +- [improvement] CASSJAVA-14: Query builder support for NOT CQL syntax +- [bug] CASSJAVA-12: DefaultSslEngineFactory missing null check on close +- [improvement] CASSJAVA-46: Expose table extensions via schema builders +- [bug] PR 1938: Fix uncaught exception during graceful channel shutdown after exceeding max orphan ids +- [improvement] PR 1607: Annotate BatchStatement, Statement, SimpleStatement methods with CheckReturnValue +- [improvement] CASSJAVA-41: Reduce lock held duration in ConcurrencyLimitingRequestThrottler +- [bug] JAVA-3149: Async Query Cancellation Not Propagated To RequestThrottler +- [bug] JAVA-3167: CompletableFutures.allSuccessful() may return never completed future +- [bug] PR 1620: Don't return empty routing key when partition key is unbound +- [improvement] PR 1623: Limit calls to Conversions.resolveExecutionProfile +- [improvement] CASSJAVA-29: Update target Cassandra versions for integration tests, support new 5.0.x + ### 4.18.1 - [improvement] JAVA-3142: Ability to specify ordering of remote local dc's via new configuration for graceful automatic failovers From 46444eaabdbd23e9231123198536d070e99aca27 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 6 Feb 2025 15:51:51 -0600 Subject: [PATCH 41/77] [maven-release-plugin] prepare release 4.19.0 --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 08f212f6157..bb39a484f71 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-core-shaded - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-mapper-processor - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-mapper-runtime - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-query-builder - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-guava-shaded - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-test-infra - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-metrics-micrometer - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-metrics-microprofile - 4.18.2-SNAPSHOT + 4.19.0 com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 9a708beb2a7..82f45050098 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index 2a48e8bf9ce..ea0efdddc08 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index ee5b52958c3..d3275565827 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index fafd8c4678b..0bb2ccef386 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index dfc406baf43..4edc44d2131 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index a76cc8d2bf1..ee90d5c2c65 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.18.2-SNAPSHOT + 4.19.0 java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index ca8f0161b04..a0627c376ff 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index d1b0a736bb0..1052f96ae35 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 6588f17d5f7..0b201db2473 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index 28483ee93ff..f846fc68dec 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 8ab939cbb37..80ec51605ab 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 521a67f9075..955160dd621 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index f0e66b656ca..7b761aa12a0 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index c61e6485fd3..ecb800b7ebb 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1029,7 +1029,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - HEAD + 4.19.0 diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 4e09a10e584..144906adecb 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 262627e5536..0252683ca2b 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-test-infra bundle From 90612f6758eb0f0ba964daf054f397a47a90a736 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 6 Feb 2025 15:51:54 -0600 Subject: [PATCH 42/77] [maven-release-plugin] prepare for next development iteration --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index bb39a484f71..8e7bc467fe7 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-core-shaded - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-mapper-processor - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-mapper-runtime - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-query-builder - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-guava-shaded - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-test-infra - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-metrics-micrometer - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-metrics-microprofile - 4.19.0 + 4.19.1-SNAPSHOT com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 82f45050098..451db1dcd1b 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index ea0efdddc08..b8d7d5c2d3b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index d3275565827..e930f4c0610 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index 0bb2ccef386..1c762074673 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index 4edc44d2131..8f7740e148f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index ee90d5c2c65..15f082e6864 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.0 + 4.19.1-SNAPSHOT java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index a0627c376ff..8053af94911 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1052f96ae35..b489076c257 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 0b201db2473..519f1411ce9 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index f846fc68dec..3a767c2a352 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 80ec51605ab..091bb5f3e93 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 955160dd621..5163b5366f4 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 7b761aa12a0..4b41f790145 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index ecb800b7ebb..3fd2d1347c2 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1029,7 +1029,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - 4.19.0 + HEAD diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 144906adecb..0bc46f9bb91 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 0252683ca2b..b0808757ce4 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-test-infra bundle From 3bb5b18903636976279506308c60f1911b7b8ed5 Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Thu, 13 Feb 2025 17:11:36 -0500 Subject: [PATCH 43/77] CASSJAVA-80: Support configuration to disable DNS reverse-lookups for SAN validation patch by Abe Ratnofsky; reviewed by Alexandre Dutra, Andy Tolbert, and Francisco Guerrero for CASSJAVA-80 --- .../api/core/config/DefaultDriverOption.java | 6 ++ .../api/core/config/TypedDriverOption.java | 4 ++ .../ssl/ProgrammaticSslEngineFactory.java | 27 +++++++- .../core/ssl/DefaultSslEngineFactory.java | 26 +++++++- .../core/ssl/SniSslEngineFactory.java | 10 ++- core/src/main/resources/reference.conf | 6 ++ .../core/ssl/DefaultSslEngineFactoryIT.java | 66 +++++++++++++++++++ 7 files changed, 141 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 11f2702c3cf..acd4bb9cc1d 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -243,6 +243,12 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: boolean */ SSL_HOSTNAME_VALIDATION("advanced.ssl-engine-factory.hostname-validation"), + /** + * Whether or not to do a DNS reverse-lookup of provided server addresses for SAN addresses. + * + *

Value-type: boolean + */ + SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"), /** * The location of the keystore file. * diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index ca60b67f0ba..93e2b468461 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -229,6 +229,10 @@ public String toString() { */ public static final TypedDriverOption SSL_HOSTNAME_VALIDATION = new TypedDriverOption<>(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, GenericType.BOOLEAN); + + public static final TypedDriverOption SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN = + new TypedDriverOption<>( + DefaultDriverOption.SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN, GenericType.BOOLEAN); /** The location of the keystore file. */ public static final TypedDriverOption SSL_KEYSTORE_PATH = new TypedDriverOption<>(DefaultDriverOption.SSL_KEYSTORE_PATH, GenericType.STRING); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/ssl/ProgrammaticSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/api/core/ssl/ProgrammaticSslEngineFactory.java index 6dfe4087b91..d65eaa864aa 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/ssl/ProgrammaticSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/ssl/ProgrammaticSslEngineFactory.java @@ -45,6 +45,7 @@ public class ProgrammaticSslEngineFactory implements SslEngineFactory { protected final SSLContext sslContext; protected final String[] cipherSuites; protected final boolean requireHostnameValidation; + protected final boolean allowDnsReverseLookupSan; /** * Creates an instance with the given {@link SSLContext}, default cipher suites and no host name @@ -80,9 +81,28 @@ public ProgrammaticSslEngineFactory( @NonNull SSLContext sslContext, @Nullable String[] cipherSuites, boolean requireHostnameValidation) { + this(sslContext, cipherSuites, requireHostnameValidation, true); + } + + /** + * Creates an instance with the given {@link SSLContext}, cipher suites and host name validation. + * + * @param sslContext the {@link SSLContext} to use. + * @param cipherSuites the cipher suites to use, or null to use the default ones. + * @param requireHostnameValidation whether to enable host name validation. If enabled, host name + * validation will be done using HTTPS algorithm. + * @param allowDnsReverseLookupSan whether to allow raw server IPs to be DNS reverse-resolved to + * choose the appropriate Subject Alternative Name. + */ + public ProgrammaticSslEngineFactory( + @NonNull SSLContext sslContext, + @Nullable String[] cipherSuites, + boolean requireHostnameValidation, + boolean allowDnsReverseLookupSan) { this.sslContext = sslContext; this.cipherSuites = cipherSuites; this.requireHostnameValidation = requireHostnameValidation; + this.allowDnsReverseLookupSan = allowDnsReverseLookupSan; } @NonNull @@ -92,7 +112,12 @@ public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { SocketAddress remoteAddress = remoteEndpoint.resolve(); if (remoteAddress instanceof InetSocketAddress) { InetSocketAddress socketAddress = (InetSocketAddress) remoteAddress; - engine = sslContext.createSSLEngine(socketAddress.getHostName(), socketAddress.getPort()); + engine = + sslContext.createSSLEngine( + allowDnsReverseLookupSan + ? socketAddress.getHostName() + : socketAddress.getHostString(), + socketAddress.getPort()); } else { engine = sslContext.createSSLEngine(); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java index 475ec38d578..343d3f9e4e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java @@ -22,6 +22,7 @@ import com.datastax.oss.driver.api.core.context.DriverContext; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.InputStream; import java.net.InetSocketAddress; @@ -69,6 +70,7 @@ public class DefaultSslEngineFactory implements SslEngineFactory { private final SSLContext sslContext; private final String[] cipherSuites; private final boolean requireHostnameValidation; + private final boolean allowDnsReverseLookupSan; private ReloadingKeyManagerFactory kmf; /** Builds a new instance from the driver configuration. */ @@ -88,6 +90,28 @@ public DefaultSslEngineFactory(DriverContext driverContext) { } this.requireHostnameValidation = config.getBoolean(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, true); + this.allowDnsReverseLookupSan = + config.getBoolean(DefaultDriverOption.SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN, true); + } + + @VisibleForTesting + protected String hostname(InetSocketAddress addr) { + return allowDnsReverseLookupSan ? hostMaybeFromDnsReverseLookup(addr) : hostNoLookup(addr); + } + + @VisibleForTesting + protected String hostMaybeFromDnsReverseLookup(InetSocketAddress addr) { + // See java.net.InetSocketAddress.getHostName: + // "This method may trigger a name service reverse lookup if the address was created with a + // literal IP address." + return addr.getHostName(); + } + + @VisibleForTesting + protected String hostNoLookup(InetSocketAddress addr) { + // See java.net.InetSocketAddress.getHostString: + // "This has the benefit of not attempting a reverse lookup" + return addr.getHostString(); } @NonNull @@ -97,7 +121,7 @@ public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { SocketAddress remoteAddress = remoteEndpoint.resolve(); if (remoteAddress instanceof InetSocketAddress) { InetSocketAddress socketAddress = (InetSocketAddress) remoteAddress; - engine = sslContext.createSSLEngine(socketAddress.getHostName(), socketAddress.getPort()); + engine = sslContext.createSSLEngine(hostname(socketAddress), socketAddress.getPort()); } else { engine = sslContext.createSSLEngine(); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java index 98af19045dc..4d2cb69fbfc 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java @@ -38,9 +38,15 @@ public class SniSslEngineFactory implements SslEngineFactory { private final SSLContext sslContext; private final CopyOnWriteArrayList fakePorts = new CopyOnWriteArrayList<>(); + private final boolean allowDnsReverseLookupSan; public SniSslEngineFactory(SSLContext sslContext) { + this(sslContext, true); + } + + public SniSslEngineFactory(SSLContext sslContext, boolean allowDnsReverseLookupSan) { this.sslContext = sslContext; + this.allowDnsReverseLookupSan = allowDnsReverseLookupSan; } @NonNull @@ -71,8 +77,8 @@ public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { // To avoid that, we create a unique "fake" port for every node. We still get session reuse for // a given node, but not across nodes. This is safe because the advisory port is only used for // session caching. - SSLEngine engine = - sslContext.createSSLEngine(address.getHostName(), getFakePort(sniServerName)); + String peerHost = allowDnsReverseLookupSan ? address.getHostName() : address.getHostString(); + SSLEngine engine = sslContext.createSSLEngine(peerHost, getFakePort(sniServerName)); engine.setUseClientMode(true); SSLParameters parameters = engine.getSSLParameters(); parameters.setServerNames(ImmutableList.of(new SNIHostName(sniServerName))); diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 7b1c43f8bea..f09ffd18a10 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -789,6 +789,12 @@ datastax-java-driver { # name matches the hostname of the server being connected to. If not set, defaults to true. // hostname-validation = true + # Whether or not to allow a DNS reverse-lookup of provided server addresses for SAN addresses, + # if cluster endpoints are specified as literal IPs. + # This is left as true for compatibility, but in most environments a DNS reverse-lookup should + # not be necessary to get an address that matches the server certificate SANs. + // allow-dns-reverse-lookup-san = true + # The locations and passwords used to access truststore and keystore contents. # These properties are optional. If either truststore-path or keystore-path are specified, # the driver builds an SSLContext from these files. If neither option is specified, the diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/ssl/DefaultSslEngineFactoryIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/ssl/DefaultSslEngineFactoryIT.java index 5f97e661eb1..a2afeade3ce 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/ssl/DefaultSslEngineFactoryIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/ssl/DefaultSslEngineFactoryIT.java @@ -21,10 +21,13 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.context.DriverContext; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.assertions.Assertions; import com.datastax.oss.driver.internal.core.ssl.DefaultSslEngineFactory; +import java.net.InetSocketAddress; import org.junit.ClassRule; import org.junit.Test; @@ -88,4 +91,67 @@ public void should_not_connect_if_not_using_ssl() { session.execute("select * from system.local"); } } + + public static class InstrumentedSslEngineFactory extends DefaultSslEngineFactory { + int countReverseLookups = 0; + int countNoLookups = 0; + + public InstrumentedSslEngineFactory(DriverContext driverContext) { + super(driverContext); + } + + @Override + protected String hostMaybeFromDnsReverseLookup(InetSocketAddress addr) { + countReverseLookups++; + return super.hostMaybeFromDnsReverseLookup(addr); + } + + @Override + protected String hostNoLookup(InetSocketAddress addr) { + countNoLookups++; + return super.hostNoLookup(addr); + } + }; + + @Test + public void should_respect_config_for_san_resolution() { + DriverConfigLoader loader = + SessionUtils.configLoaderBuilder() + .withClass( + DefaultDriverOption.SSL_ENGINE_FACTORY_CLASS, InstrumentedSslEngineFactory.class) + .withBoolean(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, false) + .withString( + DefaultDriverOption.SSL_TRUSTSTORE_PATH, + CcmBridge.DEFAULT_CLIENT_TRUSTSTORE_FILE.getAbsolutePath()) + .withString( + DefaultDriverOption.SSL_TRUSTSTORE_PASSWORD, + CcmBridge.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD) + .build(); + try (CqlSession session = SessionUtils.newSession(CCM_RULE, loader)) { + InstrumentedSslEngineFactory ssl = + (InstrumentedSslEngineFactory) session.getContext().getSslEngineFactory().get(); + Assertions.assertThat(ssl.countReverseLookups).isGreaterThan(0); + Assertions.assertThat(ssl.countNoLookups).isEqualTo(0); + } + + loader = + SessionUtils.configLoaderBuilder() + .withClass( + DefaultDriverOption.SSL_ENGINE_FACTORY_CLASS, InstrumentedSslEngineFactory.class) + .withBoolean(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, false) + .withString( + DefaultDriverOption.SSL_TRUSTSTORE_PATH, + CcmBridge.DEFAULT_CLIENT_TRUSTSTORE_FILE.getAbsolutePath()) + .withString( + DefaultDriverOption.SSL_TRUSTSTORE_PASSWORD, + CcmBridge.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD) + .withBoolean(DefaultDriverOption.SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN, false) + .build(); + try (CqlSession session = SessionUtils.newSession(CCM_RULE, loader)) { + InstrumentedSslEngineFactory ssl = + (InstrumentedSslEngineFactory) session.getContext().getSslEngineFactory().get(); + Assertions.assertThat(ssl.countReverseLookups).isEqualTo(0); + Assertions.assertThat(ssl.countNoLookups).isGreaterThan(0); + } + } } From 7982f413a90935a91549aa3ee7cc64fa25d7c113 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 3 Mar 2025 11:46:42 -0600 Subject: [PATCH 44/77] ninja-fix Minor fix to CASSJAVA-80 change. Adding new entries to DefaultDriverOption anywhere other than at the end messes with the ordinal guarantees for exisitng apps. We have a check for such a thing in the build; moving this around avoids that concern. --- .../api/core/config/DefaultDriverOption.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index acd4bb9cc1d..6ffd51d86ef 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -243,12 +243,6 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: boolean */ SSL_HOSTNAME_VALIDATION("advanced.ssl-engine-factory.hostname-validation"), - /** - * Whether or not to do a DNS reverse-lookup of provided server addresses for SAN addresses. - * - *

Value-type: boolean - */ - SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"), /** * The location of the keystore file. * @@ -994,7 +988,13 @@ public enum DefaultDriverOption implements DriverOption { *

Value type: {@link java.util.List List}<{@link String}> */ LOAD_BALANCING_DC_FAILOVER_PREFERRED_REMOTE_DCS( - "advanced.load-balancing-policy.dc-failover.preferred-remote-dcs"); + "advanced.load-balancing-policy.dc-failover.preferred-remote-dcs"), + /** + * Whether or not to do a DNS reverse-lookup of provided server addresses for SAN addresses. + * + *

Value-type: boolean + */ + SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"); private final String path; From c0cae9bf76024f8004abaa1ddb871b7351fc253e Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 25 Mar 2025 15:27:55 -0500 Subject: [PATCH 45/77] CASSJAVA-90 Update native-protocol version patch by Bret McGuire; reviewed by Abe Ratnofsky and Bret McGuire for CASSJAVA-90 --- bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bom/pom.xml b/bom/pom.xml index 8e7bc467fe7..f03317edc03 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -78,7 +78,7 @@ com.datastax.oss native-protocol - 1.5.1 + 1.5.2 From 204dd09329a073043b5b7cc170a9850e270c5a63 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 31 Mar 2025 13:24:31 -0700 Subject: [PATCH 46/77] CASSJAVA-40: Driver testing against Java 21 patch by Jane He; reviewed by Bret McGuire for CASSJAVA-40 --- Jenkinsfile-asf | 4 ++-- .../oss/driver/internal/core/util/ArrayUtils.java | 3 ++- .../internal/core/cql/QueryTraceFetcherTest.java | 2 +- .../driver/internal/core/util/ArrayUtilsTest.java | 4 ++-- pom.xml | 13 +++++++++++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf index 0217d0455d6..d6318585489 100644 --- a/Jenkinsfile-asf +++ b/Jenkinsfile-asf @@ -35,7 +35,7 @@ pipeline { axes { axis { name 'TEST_JAVA_VERSION' - values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@17' + values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@17', 'openjdk@1.21.0' } axis { name 'SERVER_VERSION' @@ -67,7 +67,7 @@ pipeline { def executeTests() { def testJavaMajorVersion = (TEST_JAVA_VERSION =~ /@(?:1\.)?(\d+)/)[0][1] sh """ - container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver apache.jfrog.io/cassan-docker/apache/cassandra-java-driver-testing-ubuntu2204 'sleep 2h') + container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver janehe158/cassandra-java-driver-dev-env 'sleep 2h') docker exec --user root \$container_id bash -c \"sudo bash /home/docker/cassandra-java-driver/ci/create-user.sh docker \$(id -u) \$(id -g) /home/docker/cassandra-java-driver\" docker exec --user docker \$container_id './cassandra-java-driver/ci/run-tests.sh' ( nohup docker stop \$container_id >/dev/null 2>/dev/null & ) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/ArrayUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/ArrayUtils.java index f5fcb98e8b7..490b1dc7d17 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/ArrayUtils.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/ArrayUtils.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.util; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Random; import java.util.concurrent.ThreadLocalRandom; public class ArrayUtils { @@ -77,7 +78,7 @@ public static void shuffleHead(@NonNull ElementT[] elements, int n) { * Fisher-Yates shuffle */ public static void shuffleHead( - @NonNull ElementT[] elements, int n, @NonNull ThreadLocalRandom random) { + @NonNull ElementT[] elements, int n, @NonNull Random random) { if (n > elements.length) { throw new ArrayIndexOutOfBoundsException( String.format( diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/QueryTraceFetcherTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/QueryTraceFetcherTest.java index b355e0fc9f0..dc238775bc1 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/QueryTraceFetcherTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/QueryTraceFetcherTest.java @@ -79,7 +79,7 @@ public class QueryTraceFetcherTest { @Mock private NettyOptions nettyOptions; @Mock private EventExecutorGroup adminEventExecutorGroup; @Mock private EventExecutor eventExecutor; - @Mock private InetAddress address; + private InetAddress address = InetAddress.getLoopbackAddress(); @Captor private ArgumentCaptor statementCaptor; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/ArrayUtilsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/ArrayUtilsTest.java index c2a7fb70304..c2df6449fdb 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/util/ArrayUtilsTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/ArrayUtilsTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.concurrent.ThreadLocalRandom; +import java.util.Random; import org.junit.Test; public class ArrayUtilsTest { @@ -86,7 +86,7 @@ public void should_not_bubble_down_when_target_index_lower() { @Test public void should_shuffle_head() { String[] array = {"a", "b", "c", "d", "e"}; - ThreadLocalRandom random = mock(ThreadLocalRandom.class); + Random random = mock(Random.class); when(random.nextInt(anyInt())) .thenAnswer( (invocation) -> { diff --git a/pom.xml b/pom.xml index 3fd2d1347c2..088d7b07532 100644 --- a/pom.xml +++ b/pom.xml @@ -1016,6 +1016,19 @@ limitations under the License.]]> --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED + + + test-jdk-21 + + [21,) + + + + -XX:+AllowRedefinitionToAddDeleteMethods + + --add-opens=java.base/jdk.internal.util.random=ALL-UNNAMED + + From 529d56e1742dcd1df3ca55c00fd8e02c0e484c68 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 31 Mar 2025 16:00:18 -0500 Subject: [PATCH 47/77] Add support for Java21 builds to test runs (plus a few other small cleanups) patch by Bret McGuire; reviewed by Joao Reis for CASSJAVA-40 --- Jenkinsfile-datastax | 50 ++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index cd48f325a29..af1aab6e0f4 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -19,22 +19,15 @@ */ def initializeEnvironment() { - env.DRIVER_DISPLAY_NAME = 'CassandraⓇ Java Driver' + env.DRIVER_DISPLAY_NAME = 'Java Driver for Apache CassandraⓇ' env.DRIVER_METRIC_TYPE = 'oss' - if (env.GIT_URL.contains('riptano/java-driver')) { - env.DRIVER_DISPLAY_NAME = 'private ' + env.DRIVER_DISPLAY_NAME - env.DRIVER_METRIC_TYPE = 'oss-private' - } else if (env.GIT_URL.contains('java-dse-driver')) { - env.DRIVER_DISPLAY_NAME = 'DSE Java Driver' - env.DRIVER_METRIC_TYPE = 'dse' - } env.GIT_SHA = "${env.GIT_COMMIT.take(7)}" env.GITHUB_PROJECT_URL = "https://${GIT_URL.replaceFirst(/(git@|http:\/\/|https:\/\/)/, '').replace(':', '/').replace('.git', '')}" env.GITHUB_BRANCH_URL = "${GITHUB_PROJECT_URL}/tree/${env.BRANCH_NAME}" env.GITHUB_COMMIT_URL = "${GITHUB_PROJECT_URL}/commit/${env.GIT_COMMIT}" - env.MAVEN_HOME = "${env.HOME}/.mvn/apache-maven-3.3.9" + env.MAVEN_HOME = "${env.HOME}/.mvn/apache-maven-3.6.3" env.PATH = "${env.MAVEN_HOME}/bin:${env.PATH}" /* @@ -335,14 +328,12 @@ pipeline { ''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_JABBA_VERSION', - choices: ['1.8', // Oracle JDK version 1.8 (current default) - 'openjdk@1.9', // OpenJDK version 9 - 'openjdk@1.10', // OpenJDK version 10 + choices: [ + '1.8', // Oracle JDK version 1.8 (current default) 'openjdk@1.11', // OpenJDK version 11 - 'openjdk@1.12', // OpenJDK version 12 - 'openjdk@1.13', // OpenJDK version 13 - 'openjdk@1.14', // OpenJDK version 14 - 'openjdk@1.17'], // OpenJDK version 17 + 'openjdk@1.17', // OpenJDK version 17 + 'openjdk@1.21' // OpenJDK version 21 + ], description: '''JDK version to use for TESTING when running adhoc BUILD-AND-EXECUTE-TESTS builds. All builds will use JDK8 for building the driver @@ -355,34 +346,18 @@ pipeline { - - - - - - - - - - - - - - - - - - - - + + + +
1.8 Oracle JDK version 1.8 (Used for compiling regardless of choice)
openjdk@1.9OpenJDK version 9
openjdk@1.10OpenJDK version 10
openjdk@1.11 OpenJDK version 11
openjdk@1.12OpenJDK version 12
openjdk@1.13OpenJDK version 13
openjdk@1.14OpenJDK version 14
openjdk@1.17 OpenJDK version 17
openjdk@1.21OpenJDK version 21
''') booleanParam( name: 'SKIP_SERIAL_ITS', @@ -466,7 +441,8 @@ pipeline { name 'JABBA_VERSION' values '1.8', // jdk8 'openjdk@1.11', // jdk11 - 'openjdk@1.17' // jdk17 + 'openjdk@1.17', // jdk17 + 'openjdk@1.21' // jdk21 } } From f42ab99ccc031bca7db6cf69aec70771d65799e1 Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Mon, 10 Feb 2025 13:11:06 -0500 Subject: [PATCH 48/77] Upgrade Netty to 4.1.119 patch by Abe Ratnofsky; reviewed by Bret McGuire for CASSJAVA-77 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 088d7b07532..a8841a9caca 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 2.1.12 4.1.18 - 4.1.94.Final + 4.1.119.Final 1.2.1 - com.datastax.oss:${project.artifactId}:RELEASE + ${project.groupId}:${project.artifactId}:RELEASE From 0115cd67c18835b89d8888d405d455045737a630 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 11 Apr 2025 13:19:45 +0200 Subject: [PATCH 50/77] Prevent long overflow in SNI address resolution patch by Lukasz Antoniak and Alexandre Dutra; reviewed by Bret McGuire --- .../oss/driver/internal/core/metadata/SniEndPoint.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index ace4e82617d..d1ab8eec98d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -26,10 +26,10 @@ import java.util.Arrays; import java.util.Comparator; import java.util.Objects; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicInteger; public class SniEndPoint implements EndPoint { - private static final AtomicLong OFFSET = new AtomicLong(); + private static final AtomicInteger OFFSET = new AtomicInteger(); private final InetSocketAddress proxyAddress; private final String serverName; @@ -64,7 +64,10 @@ public InetSocketAddress resolve() { // The order of the returned address is unspecified. Sort by IP to make sure we get a true // round-robin Arrays.sort(aRecords, IP_COMPARATOR); - int index = (aRecords.length == 1) ? 0 : (int) OFFSET.getAndIncrement() % aRecords.length; + int index = + (aRecords.length == 1) + ? 0 + : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; return new InetSocketAddress(aRecords[index], proxyAddress.getPort()); } catch (UnknownHostException e) { throw new IllegalArgumentException( From 7161d12fb28d16e2928cfe98bbada151ff6ae058 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 23 Apr 2025 22:25:03 -0700 Subject: [PATCH 51/77] ninja-fix Revert docker image name (CASSJAVA-40) --- Jenkinsfile-asf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf index d6318585489..24800ba9051 100644 --- a/Jenkinsfile-asf +++ b/Jenkinsfile-asf @@ -67,7 +67,7 @@ pipeline { def executeTests() { def testJavaMajorVersion = (TEST_JAVA_VERSION =~ /@(?:1\.)?(\d+)/)[0][1] sh """ - container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver janehe158/cassandra-java-driver-dev-env 'sleep 2h') + container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver apache.jfrog.io/cassan-docker/apache/cassandra-java-driver-testing-ubuntu2204 'sleep 2h') docker exec --user root \$container_id bash -c \"sudo bash /home/docker/cassandra-java-driver/ci/create-user.sh docker \$(id -u) \$(id -g) /home/docker/cassandra-java-driver\" docker exec --user docker \$container_id './cassandra-java-driver/ci/run-tests.sh' ( nohup docker stop \$container_id >/dev/null 2>/dev/null & ) From d7e829775c4956d10c888c86b653f7ac2d10fa4b Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:57:52 -0600 Subject: [PATCH 52/77] Make guava an optional dependency of java-driver-guava-shaded With CASSJAVA-52, the java-driver-guava-shaded is now in tree. This appears to work great, but there is a slight issue with the dependency tree that allows unshaded guava packages to be imported within the project. It looks like marking guava as an optional dependency in java-driver-guava-shaded resolves this. patch by Andy Tolbert; reviewed by Alexandre Dutra and Dmitry Konstantinov for CASSJAVA-76 --- guava-shaded/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 8053af94911..ed37e861a96 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -46,6 +46,7 @@ error_prone_annotations + true org.graalvm.nativeimage From eb54934d5d2f38757da3a94a8ad4960f66eb4bd6 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 15 Nov 2024 19:01:32 -0800 Subject: [PATCH 53/77] Bump Jackson version to la(te)st 2.13, 2.13.5 patch by Tatu Saloranta; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1989 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c69dabb84ce..e5cfb58f94d 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ 1.0.3 20230227 - 2.13.4 - 2.13.4.2 + 2.13.5 + ${jackson.version} 1.1.10.1 1.7.1 From 53bb5c8b6580c5797e4f148cdc818885327cd19e Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 15 Nov 2024 18:41:50 -0800 Subject: [PATCH 54/77] CASSJAVA-68 Improve DefaultCodecRegisry.CacheKey#hashCode() to eliminate Object[] allocation (found via profiler) patch by Tatu Saloranta; reviewed by Dmitry Konstantinov and Bret McGuire for CASSJAVA-68 --- .../core/type/codec/registry/DefaultCodecRegistry.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java index cfd053ea56e..4334f22b63d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java @@ -159,7 +159,10 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hash(cqlType, javaType, isJavaCovariant); + // NOTE: inlined Objects.hash for performance reasons (avoid Object[] allocation + // seen in profiler allocation traces) + return ((31 + Objects.hashCode(cqlType)) * 31 + Objects.hashCode(javaType)) * 31 + + Boolean.hashCode(isJavaCovariant); } } } From c9facc3c36e7ba0b1d30fb9b10de69f879d34fb5 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 5 May 2025 19:37:32 -0700 Subject: [PATCH 55/77] ninja-fix Format fix for previous commit --- .../internal/core/type/codec/registry/DefaultCodecRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java index 4334f22b63d..cc14740e180 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/DefaultCodecRegistry.java @@ -162,7 +162,7 @@ public int hashCode() { // NOTE: inlined Objects.hash for performance reasons (avoid Object[] allocation // seen in profiler allocation traces) return ((31 + Objects.hashCode(cqlType)) * 31 + Objects.hashCode(javaType)) * 31 - + Boolean.hashCode(isJavaCovariant); + + Boolean.hashCode(isJavaCovariant); } } } From bb9bb11e573258e12e4272379f88e4d8b5f103a0 Mon Sep 17 00:00:00 2001 From: alexsa Date: Thu, 12 Jun 2025 10:02:09 +0200 Subject: [PATCH 56/77] Add SubnetAddressTranslator to translate Cassandra node IPs from private network based on its subnet mask patch by Alex Sasnouskikh; reviewed by Bret McGuire and Andy Tolbert reference: https://github.com/apache/cassandra-java-driver/pull/2013 --- .../api/core/config/DefaultDriverOption.java | 43 ++++- .../api/core/config/TypedDriverOption.java | 14 ++ .../driver/internal/core/ContactPoints.java | 61 ++---- .../FixedHostNameAddressTranslator.java | 20 +- .../core/addresstranslation/Subnet.java | 176 ++++++++++++++++++ .../addresstranslation/SubnetAddress.java | 65 +++++++ .../SubnetAddressTranslator.java | 148 +++++++++++++++ .../internal/core/util/AddressUtils.java | 59 ++++++ core/src/main/resources/reference.conf | 18 +- .../internal/core/ContactPointsTest.java | 4 +- .../FixedHostNameAddressTranslatorTest.java | 5 +- .../addresstranslation/SubnetAddressTest.java | 44 +++++ .../SubnetAddressTranslatorTest.java | 153 +++++++++++++++ .../core/addresstranslation/SubnetTest.java | 118 ++++++++++++ .../internal/core/config/MockOptions.java | 1 + .../typesafe/TypesafeDriverConfigTest.java | 14 +- manual/core/address_resolution/README.md | 49 +++++ 17 files changed, 923 insertions(+), 69 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Subnet.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddress.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslator.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 6ffd51d86ef..4e45bf7b117 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -994,7 +994,48 @@ public enum DefaultDriverOption implements DriverOption { * *

Value-type: boolean */ - SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"); + SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"), + /** + * An address to always translate all node addresses to that same proxy hostname no matter what IP + * address a node has, but still using its native transport port. + * + *

Value-Type: {@link String} + */ + ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME("advanced.address-translator.advertised-hostname"), + /** + * A map of Cassandra node subnets (CIDR notations) to target addresses, for example (note quoted + * keys): + * + * 

+   * advanced.address-translator.subnet-addresses {
+   *   "100.64.0.0/15" = "cassandra.datacenter1.com:9042"
+   *   "100.66.0.0/15" = "cassandra.datacenter2.com:9042"
+   *   # IPv6 example:
+   *   # "::ffff:6440:0/111" = "cassandra.datacenter1.com:9042"
+   *   # "::ffff:6442:0/111" = "cassandra.datacenter2.com:9042"
+   * }
+   *
+ * + * Note: subnets must be represented as prefix blocks, see {@link + * inet.ipaddr.Address#isPrefixBlock()}. + * + *

Value type: {@link java.util.Map Map}<{@link String},{@link String}> + */ + ADDRESS_TRANSLATOR_SUBNET_ADDRESSES("advanced.address-translator.subnet-addresses"), + /** + * A default address to fallback to if Cassandra node IP isn't contained in any of the configured + * subnets. + * + *

Value-Type: {@link String} + */ + ADDRESS_TRANSLATOR_DEFAULT_ADDRESS("advanced.address-translator.default-address"), + /** + * Whether to resolve the addresses on initialization (if true) or on each node (re-)connection + * (if false). Defaults to false. + * + *

Value-Type: boolean + */ + ADDRESS_TRANSLATOR_RESOLVE_ADDRESSES("advanced.address-translator.resolve-addresses"); private final String path; diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index 93e2b468461..aa4e4af12dc 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -896,6 +896,20 @@ public String toString() { DefaultDriverOption.LOAD_BALANCING_DC_FAILOVER_ALLOW_FOR_LOCAL_CONSISTENCY_LEVELS, GenericType.BOOLEAN); + public static final TypedDriverOption ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME = + new TypedDriverOption<>( + DefaultDriverOption.ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME, GenericType.STRING); + public static final TypedDriverOption> ADDRESS_TRANSLATOR_SUBNET_ADDRESSES = + new TypedDriverOption<>( + DefaultDriverOption.ADDRESS_TRANSLATOR_SUBNET_ADDRESSES, + GenericType.mapOf(GenericType.STRING, GenericType.STRING)); + public static final TypedDriverOption ADDRESS_TRANSLATOR_DEFAULT_ADDRESS = + new TypedDriverOption<>( + DefaultDriverOption.ADDRESS_TRANSLATOR_DEFAULT_ADDRESS, GenericType.STRING); + public static final TypedDriverOption ADDRESS_TRANSLATOR_RESOLVE_ADDRESSES = + new TypedDriverOption<>( + DefaultDriverOption.ADDRESS_TRANSLATOR_RESOLVE_ADDRESSES, GenericType.BOOLEAN); + /** * Ordered preference list of remote dcs optionally supplied for automatic failover and included * in query plan. This feature is enabled only when max-nodes-per-remote-dc is greater than 0. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ContactPoints.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ContactPoints.java index 1ed2a1cebf3..bb65661b72f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ContactPoints.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ContactPoints.java @@ -19,14 +19,11 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.util.AddressUtils; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; import com.datastax.oss.driver.shaded.guava.common.collect.Sets; -import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; import org.slf4j.Logger; @@ -41,7 +38,22 @@ public static Set merge( Set result = Sets.newHashSet(programmaticContactPoints); for (String spec : configContactPoints) { - for (InetSocketAddress address : extract(spec, resolve)) { + + Set addresses = Collections.emptySet(); + try { + addresses = AddressUtils.extract(spec, resolve); + } catch (RuntimeException e) { + LOG.warn("Ignoring invalid contact point {} ({})", spec, e.getMessage(), e); + } + + if (addresses.size() > 1) { + LOG.info( + "Contact point {} resolves to multiple addresses, will use them all ({})", + spec, + addresses); + } + + for (InetSocketAddress address : addresses) { DefaultEndPoint endPoint = new DefaultEndPoint(address); boolean wasNew = result.add(endPoint); if (!wasNew) { @@ -51,43 +63,4 @@ public static Set merge( } return ImmutableSet.copyOf(result); } - - private static Set extract(String spec, boolean resolve) { - int separator = spec.lastIndexOf(':'); - if (separator < 0) { - LOG.warn("Ignoring invalid contact point {} (expecting host:port)", spec); - return Collections.emptySet(); - } - - String host = spec.substring(0, separator); - String portSpec = spec.substring(separator + 1); - int port; - try { - port = Integer.parseInt(portSpec); - } catch (NumberFormatException e) { - LOG.warn("Ignoring invalid contact point {} (expecting a number, got {})", spec, portSpec); - return Collections.emptySet(); - } - if (!resolve) { - return ImmutableSet.of(InetSocketAddress.createUnresolved(host, port)); - } else { - try { - InetAddress[] inetAddresses = InetAddress.getAllByName(host); - if (inetAddresses.length > 1) { - LOG.info( - "Contact point {} resolves to multiple addresses, will use them all ({})", - spec, - Arrays.deepToString(inetAddresses)); - } - Set result = new HashSet<>(); - for (InetAddress inetAddress : inetAddresses) { - result.add(new InetSocketAddress(inetAddress, port)); - } - return result; - } catch (UnknownHostException e) { - LOG.warn("Ignoring invalid contact point {} (unknown host {})", spec, host); - return Collections.emptySet(); - } - } - } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java index 4fb9782f566..5cc6c2518fb 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java @@ -17,8 +17,9 @@ */ package com.datastax.oss.driver.internal.core.addresstranslation; +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME; + import com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator; -import com.datastax.oss.driver.api.core.config.DriverOption; import com.datastax.oss.driver.api.core.context.DriverContext; import edu.umd.cs.findbugs.annotations.NonNull; import java.net.InetSocketAddress; @@ -37,28 +38,13 @@ public class FixedHostNameAddressTranslator implements AddressTranslator { private static final Logger LOG = LoggerFactory.getLogger(FixedHostNameAddressTranslator.class); - public static final String ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME = - "advanced.address-translator.advertised-hostname"; - - public static DriverOption ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME_OPTION = - new DriverOption() { - @NonNull - @Override - public String getPath() { - return ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME; - } - }; - private final String advertisedHostname; private final String logPrefix; public FixedHostNameAddressTranslator(@NonNull DriverContext context) { logPrefix = context.getSessionName(); advertisedHostname = - context - .getConfig() - .getDefaultProfile() - .getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME_OPTION); + context.getConfig().getDefaultProfile().getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME); } @NonNull diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Subnet.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Subnet.java new file mode 100644 index 00000000000..7c25e94e2f9 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Subnet.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import com.datastax.oss.driver.shaded.guava.common.base.Splitter; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; + +class Subnet { + private final byte[] subnet; + private final byte[] networkMask; + private final byte[] upper; + private final byte[] lower; + + private Subnet(byte[] subnet, byte[] networkMask) { + this.subnet = subnet; + this.networkMask = networkMask; + + byte[] upper = new byte[subnet.length]; + byte[] lower = new byte[subnet.length]; + for (int i = 0; i < subnet.length; i++) { + upper[i] = (byte) (subnet[i] | ~networkMask[i]); + lower[i] = (byte) (subnet[i] & networkMask[i]); + } + this.upper = upper; + this.lower = lower; + } + + static Subnet parse(String subnetCIDR) throws UnknownHostException { + List parts = Splitter.on("/").splitToList(subnetCIDR); + if (parts.size() != 2) { + throw new IllegalArgumentException("Invalid subnet: " + subnetCIDR); + } + + boolean isIPv6 = parts.get(0).contains(":"); + byte[] subnet = InetAddress.getByName(parts.get(0)).getAddress(); + if (isIPv4(subnet) && isIPv6) { + subnet = toIPv6(subnet); + } + int prefixLength = Integer.parseInt(parts.get(1)); + validatePrefixLength(subnet, prefixLength); + + byte[] networkMask = toNetworkMask(subnet, prefixLength); + validateSubnetIsPrefixBlock(subnet, networkMask, subnetCIDR); + return new Subnet(subnet, networkMask); + } + + private static byte[] toNetworkMask(byte[] subnet, int prefixLength) { + int fullBytes = prefixLength / 8; + int remainingBits = prefixLength % 8; + byte[] mask = new byte[subnet.length]; + Arrays.fill(mask, 0, fullBytes, (byte) 0xFF); + if (remainingBits > 0) { + mask[fullBytes] = (byte) (0xFF << (8 - remainingBits)); + } + return mask; + } + + private static void validatePrefixLength(byte[] subnet, int prefixLength) { + int max_prefix_length = subnet.length * 8; + if (prefixLength < 0 || max_prefix_length < prefixLength) { + throw new IllegalArgumentException( + String.format( + "Prefix length %s must be within [0; %s]", prefixLength, max_prefix_length)); + } + } + + private static void validateSubnetIsPrefixBlock( + byte[] subnet, byte[] networkMask, String subnetCIDR) { + byte[] prefixBlock = toPrefixBlock(subnet, networkMask); + if (!Arrays.equals(subnet, prefixBlock)) { + throw new IllegalArgumentException( + String.format("Subnet %s must be represented as a network prefix block", subnetCIDR)); + } + } + + private static byte[] toPrefixBlock(byte[] subnet, byte[] networkMask) { + byte[] prefixBlock = new byte[subnet.length]; + for (int i = 0; i < subnet.length; i++) { + prefixBlock[i] = (byte) (subnet[i] & networkMask[i]); + } + return prefixBlock; + } + + @VisibleForTesting + byte[] getSubnet() { + return Arrays.copyOf(subnet, subnet.length); + } + + @VisibleForTesting + byte[] getNetworkMask() { + return Arrays.copyOf(networkMask, networkMask.length); + } + + byte[] getUpper() { + return Arrays.copyOf(upper, upper.length); + } + + byte[] getLower() { + return Arrays.copyOf(lower, lower.length); + } + + boolean isIPv4() { + return isIPv4(subnet); + } + + boolean isIPv6() { + return isIPv6(subnet); + } + + boolean contains(byte[] ip) { + if (isIPv4() && !isIPv4(ip)) { + return false; + } + if (isIPv6() && isIPv4(ip)) { + ip = toIPv6(ip); + } + if (subnet.length != ip.length) { + throw new IllegalArgumentException( + "IP version is unknown: " + Arrays.toString(toZeroBasedByteArray(ip))); + } + for (int i = 0; i < subnet.length; i++) { + if (subnet[i] != (byte) (ip[i] & networkMask[i])) { + return false; + } + } + return true; + } + + private static boolean isIPv4(byte[] ip) { + return ip.length == 4; + } + + private static boolean isIPv6(byte[] ip) { + return ip.length == 16; + } + + private static byte[] toIPv6(byte[] ipv4) { + byte[] ipv6 = new byte[16]; + ipv6[10] = (byte) 0xFF; + ipv6[11] = (byte) 0xFF; + System.arraycopy(ipv4, 0, ipv6, 12, 4); + return ipv6; + } + + @Override + public String toString() { + return Arrays.toString(toZeroBasedByteArray(subnet)); + } + + private static int[] toZeroBasedByteArray(byte[] bytes) { + int[] res = new int[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + res[i] = bytes[i] & 0xFF; + } + return res; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddress.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddress.java new file mode 100644 index 00000000000..105e776a507 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddress.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + +class SubnetAddress { + private final Subnet subnet; + private final InetSocketAddress address; + + SubnetAddress(String subnetCIDR, InetSocketAddress address) { + try { + this.subnet = Subnet.parse(subnetCIDR); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + this.address = address; + } + + InetSocketAddress getAddress() { + return this.address; + } + + boolean isOverlapping(SubnetAddress other) { + Subnet thisSubnet = this.subnet; + Subnet otherSubnet = other.subnet; + return thisSubnet.contains(otherSubnet.getLower()) + || thisSubnet.contains(otherSubnet.getUpper()) + || otherSubnet.contains(thisSubnet.getLower()) + || otherSubnet.contains(thisSubnet.getUpper()); + } + + boolean contains(InetSocketAddress address) { + return subnet.contains(address.getAddress().getAddress()); + } + + boolean isIPv4() { + return subnet.isIPv4(); + } + + boolean isIPv6() { + return subnet.isIPv6(); + } + + @Override + public String toString() { + return "SubnetAddress[subnet=" + subnet + ", address=" + address + "]"; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslator.java new file mode 100644 index 00000000000..85f29e3fadd --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslator.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_DEFAULT_ADDRESS; +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_RESOLVE_ADDRESSES; +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_SUBNET_ADDRESSES; + +import com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator; +import com.datastax.oss.driver.api.core.context.DriverContext; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This translator returns the proxy address of the private subnet containing the Cassandra node IP, + * or default address if no matching subnets, or passes through the original node address if no + * default configured. + * + *

The translator can be used for scenarios when all nodes are behind some kind of proxy, and + * that proxy is different for nodes located in different subnets (eg. when Cassandra is deployed in + * multiple datacenters/regions). One can use this, for example, for Cassandra on Kubernetes with + * different Cassandra datacenters deployed to different Kubernetes clusters. + */ +public class SubnetAddressTranslator implements AddressTranslator { + private static final Logger LOG = LoggerFactory.getLogger(SubnetAddressTranslator.class); + + private final List subnetAddresses; + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private final Optional defaultAddress; + + private final String logPrefix; + + public SubnetAddressTranslator(@NonNull DriverContext context) { + logPrefix = context.getSessionName(); + boolean resolveAddresses = + context + .getConfig() + .getDefaultProfile() + .getBoolean(ADDRESS_TRANSLATOR_RESOLVE_ADDRESSES, false); + this.subnetAddresses = + context.getConfig().getDefaultProfile().getStringMap(ADDRESS_TRANSLATOR_SUBNET_ADDRESSES) + .entrySet().stream() + .map( + e -> { + // Quoted and/or containing forward slashes map keys in reference.conf are read to + // strings with additional quotes, eg. 100.64.0.0/15 -> '100.64.0."0/15"' or + // "100.64.0.0/15" -> '"100.64.0.0/15"' + String subnetCIDR = e.getKey().replaceAll("\"", ""); + String address = e.getValue(); + return new SubnetAddress(subnetCIDR, parseAddress(address, resolveAddresses)); + }) + .collect(Collectors.toList()); + this.defaultAddress = + Optional.ofNullable( + context + .getConfig() + .getDefaultProfile() + .getString(ADDRESS_TRANSLATOR_DEFAULT_ADDRESS, null)) + .map(address -> parseAddress(address, resolveAddresses)); + + validateSubnetsAreOfSameProtocol(this.subnetAddresses); + validateSubnetsAreNotOverlapping(this.subnetAddresses); + } + + private static void validateSubnetsAreOfSameProtocol(List subnets) { + for (int i = 0; i < subnets.size() - 1; i++) { + for (int j = i + 1; j < subnets.size(); j++) { + SubnetAddress subnet1 = subnets.get(i); + SubnetAddress subnet2 = subnets.get(j); + if (subnet1.isIPv4() != subnet2.isIPv4() && subnet1.isIPv6() != subnet2.isIPv6()) { + throw new IllegalArgumentException( + String.format( + "Configured subnets are of the different protocols: %s, %s", subnet1, subnet2)); + } + } + } + } + + private static void validateSubnetsAreNotOverlapping(List subnets) { + for (int i = 0; i < subnets.size() - 1; i++) { + for (int j = i + 1; j < subnets.size(); j++) { + SubnetAddress subnet1 = subnets.get(i); + SubnetAddress subnet2 = subnets.get(j); + if (subnet1.isOverlapping(subnet2)) { + throw new IllegalArgumentException( + String.format("Configured subnets are overlapping: %s, %s", subnet1, subnet2)); + } + } + } + } + + @NonNull + @Override + public InetSocketAddress translate(@NonNull InetSocketAddress address) { + InetSocketAddress translatedAddress = null; + for (SubnetAddress subnetAddress : subnetAddresses) { + if (subnetAddress.contains(address)) { + translatedAddress = subnetAddress.getAddress(); + } + } + if (translatedAddress == null && defaultAddress.isPresent()) { + translatedAddress = defaultAddress.get(); + } + if (translatedAddress == null) { + translatedAddress = address; + } + LOG.debug("[{}] Translated {} to {}", logPrefix, address, translatedAddress); + return translatedAddress; + } + + @Override + public void close() {} + + @Nullable + private InetSocketAddress parseAddress(String address, boolean resolve) { + try { + InetSocketAddress parsedAddress = AddressUtils.extract(address, resolve).iterator().next(); + LOG.debug("[{}] Parsed {} to {}", logPrefix, address, parsedAddress); + return parsedAddress; + } catch (RuntimeException e) { + throw new IllegalArgumentException( + String.format("Invalid address %s (%s)", address, e.getMessage()), e); + } + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java new file mode 100644 index 00000000000..8905edb9192 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.util; + +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; + +public class AddressUtils { + + public static Set extract(String address, boolean resolve) { + int separator = address.lastIndexOf(':'); + if (separator < 0) { + throw new IllegalArgumentException("expecting format host:port"); + } + + String host = address.substring(0, separator); + String portString = address.substring(separator + 1); + int port; + try { + port = Integer.parseInt(portString); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("expecting port to be a number, got " + portString, e); + } + if (!resolve) { + return ImmutableSet.of(InetSocketAddress.createUnresolved(host, port)); + } else { + InetAddress[] inetAddresses; + try { + inetAddresses = InetAddress.getAllByName(host); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + Set result = new HashSet<>(); + for (InetAddress inetAddress : inetAddresses) { + result.add(new InetSocketAddress(inetAddress, port)); + } + return result; + } + } +} diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index f09ffd18a10..3c6851a48ee 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1026,8 +1026,9 @@ datastax-java-driver { # the package com.datastax.oss.driver.internal.core.addresstranslation. # # The driver provides the following implementations out of the box: - # - PassThroughAddressTranslator: returns all addresses unchanged + # - PassThroughAddressTranslator: returns all addresses unchanged. # - FixedHostNameAddressTranslator: translates all addresses to a specific hostname. + # - SubnetAddressTranslator: translates addresses to hostname based on the subnet match. # - Ec2MultiRegionAddressTranslator: suitable for an Amazon multi-region EC2 deployment where # clients are also deployed in EC2. It optimizes network costs by favoring private IPs over # public ones whenever possible. @@ -1035,8 +1036,23 @@ datastax-java-driver { # You can also specify a custom class that implements AddressTranslator and has a public # constructor with a DriverContext argument. class = PassThroughAddressTranslator + # # This property has to be set only in case you use FixedHostNameAddressTranslator. # advertised-hostname = mycustomhostname + # + # These properties are only applicable in case you use SubnetAddressTranslator. + # subnet-addresses { + # "100.64.0.0/15" = "cassandra.datacenter1.com:9042" + # "100.66.0.0/15" = "cassandra.datacenter2.com:9042" + # # IPv6 example: + # # "::ffff:6440:0/111" = "cassandra.datacenter1.com:9042" + # # "::ffff:6442:0/111" = "cassandra.datacenter2.com:9042" + # } + # Optional. When configured, addresses not matching the configured subnets are translated to this address. + # default-address = "cassandra.datacenter1.com:9042" + # Whether to resolve the addresses once on initialization (if true) or on each node (re-)connection (if false). + # If not configured, defaults to false. + # resolve-addresses = false } # Whether to resolve the addresses passed to `basic.contact-points`. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/ContactPointsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/ContactPointsTest.java index 9e0d8737619..72b875b8602 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/ContactPointsTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/ContactPointsTest.java @@ -121,7 +121,7 @@ public void should_ignore_malformed_host_and_port_and_warn() { ContactPoints.merge(Collections.emptySet(), ImmutableList.of("foobar"), true); assertThat(endPoints).isEmpty(); - assertLog(Level.WARN, "Ignoring invalid contact point foobar (expecting host:port)"); + assertLog(Level.WARN, "Ignoring invalid contact point foobar (expecting format host:port)"); } @Test @@ -132,7 +132,7 @@ public void should_ignore_malformed_port_and_warn() { assertThat(endPoints).isEmpty(); assertLog( Level.WARN, - "Ignoring invalid contact point 127.0.0.1:foobar (expecting a number, got foobar)"); + "Ignoring invalid contact point 127.0.0.1:foobar (expecting port to be a number, got foobar)"); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java index c5e864b4bae..92800998056 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java @@ -17,6 +17,7 @@ */ package com.datastax.oss.driver.internal.core.addresstranslation; +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -33,9 +34,7 @@ public class FixedHostNameAddressTranslatorTest { @Test public void should_translate_address() { DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); - when(defaultProfile.getString( - FixedHostNameAddressTranslator.ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME_OPTION)) - .thenReturn("myaddress"); + when(defaultProfile.getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME)).thenReturn("myaddress"); DefaultDriverContext defaultDriverContext = MockedDriverContextFactory.defaultDriverContext(Optional.of(defaultProfile)); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTest.java new file mode 100644 index 00000000000..bd505f5dd44 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.net.InetSocketAddress; +import org.junit.Test; + +public class SubnetAddressTest { + @Test + public void should_return_return_true_on_overlapping_with_another_subnet_address() { + SubnetAddress subnetAddress1 = + new SubnetAddress("100.64.0.0/15", mock(InetSocketAddress.class)); + SubnetAddress subnetAddress2 = + new SubnetAddress("100.65.0.0/16", mock(InetSocketAddress.class)); + assertThat(subnetAddress1.isOverlapping(subnetAddress2)).isTrue(); + } + + @Test + public void should_return_return_false_on_not_overlapping_with_another_subnet_address() { + SubnetAddress subnetAddress1 = + new SubnetAddress("100.64.0.0/15", mock(InetSocketAddress.class)); + SubnetAddress subnetAddress2 = + new SubnetAddress("100.66.0.0/15", mock(InetSocketAddress.class)); + assertThat(subnetAddress1.isOverlapping(subnetAddress2)).isFalse(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java new file mode 100644 index 00000000000..2aa6ae75bc2 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_DEFAULT_ADDRESS; +import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_SUBNET_ADDRESSES; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.internal.core.context.DefaultDriverContext; +import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.Optional; +import org.junit.Test; + +@SuppressWarnings("resource") +public class SubnetAddressTranslatorTest { + + @Test + public void should_translate_to_correct_subnet_address_ipv4() { + Map subnetAddresses = + ImmutableMap.of( + "\"100.64.0.0/15\"", "cassandra.datacenter1.com:19042", + "100.66.0.\"0/15\"", "cassandra.datacenter2.com:19042"); + DefaultDriverContext context = context(subnetAddresses); + SubnetAddressTranslator translator = new SubnetAddressTranslator(context); + InetSocketAddress address = new InetSocketAddress("100.64.0.1", 9042); + assertThat(translator.translate(address)) + .isEqualTo(InetSocketAddress.createUnresolved("cassandra.datacenter1.com", 19042)); + } + + @Test + public void should_translate_to_correct_subnet_address_ipv6() { + Map subnetAddresses = + ImmutableMap.of( + "\"::ffff:6440:0/111\"", "cassandra.datacenter1.com:19042", + "\"::ffff:6442:0/111\"", "cassandra.datacenter2.com:19042"); + DefaultDriverContext context = context(subnetAddresses); + SubnetAddressTranslator translator = new SubnetAddressTranslator(context); + InetSocketAddress address = new InetSocketAddress("::ffff:6440:1", 9042); + assertThat(translator.translate(address)) + .isEqualTo(InetSocketAddress.createUnresolved("cassandra.datacenter1.com", 19042)); + } + + @Test + public void should_translate_to_default_address() { + DefaultDriverContext context = context(ImmutableMap.of()); + when(context + .getConfig() + .getDefaultProfile() + .getString(ADDRESS_TRANSLATOR_DEFAULT_ADDRESS, null)) + .thenReturn("cassandra.com:19042"); + SubnetAddressTranslator translator = new SubnetAddressTranslator(context); + InetSocketAddress address = new InetSocketAddress("100.68.0.1", 9042); + assertThat(translator.translate(address)) + .isEqualTo(InetSocketAddress.createUnresolved("cassandra.com", 19042)); + } + + @Test + public void should_pass_through_not_matched_address() { + DefaultDriverContext context = context(ImmutableMap.of()); + SubnetAddressTranslator translator = new SubnetAddressTranslator(context); + InetSocketAddress address = new InetSocketAddress("100.68.0.1", 9042); + assertThat(translator.translate(address)).isEqualTo(address); + } + + @Test + public void should_fail_on_intersecting_subnets_ipv4() { + Map subnetAddresses = + ImmutableMap.of( + "\"100.64.0.0/15\"", "cassandra.datacenter1.com:19042", + "100.65.0.\"0/16\"", "cassandra.datacenter2.com:19042"); + DefaultDriverContext context = context(subnetAddresses); + assertThatIllegalArgumentException() + .isThrownBy(() -> new SubnetAddressTranslator(context)) + .withMessage( + "Configured subnets are overlapping: " + + String.format( + "SubnetAddress[subnet=[100, 64, 0, 0], address=%s], ", + InetSocketAddress.createUnresolved("cassandra.datacenter1.com", 19042)) + + String.format( + "SubnetAddress[subnet=[100, 65, 0, 0], address=%s]", + InetSocketAddress.createUnresolved("cassandra.datacenter2.com", 19042))); + } + + @Test + public void should_fail_on_intersecting_subnets_ipv6() { + Map subnetAddresses = + ImmutableMap.of( + "\"::ffff:6440:0/111\"", "cassandra.datacenter1.com:19042", + "\"::ffff:6441:0/112\"", "cassandra.datacenter2.com:19042"); + DefaultDriverContext context = context(subnetAddresses); + assertThatIllegalArgumentException() + .isThrownBy(() -> new SubnetAddressTranslator(context)) + .withMessage( + "Configured subnets are overlapping: " + + String.format( + "SubnetAddress[subnet=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 100, 64, 0, 0], address=%s], ", + InetSocketAddress.createUnresolved("cassandra.datacenter1.com", 19042)) + + String.format( + "SubnetAddress[subnet=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 100, 65, 0, 0], address=%s]", + InetSocketAddress.createUnresolved("cassandra.datacenter2.com", 19042))); + } + + @Test + public void should_fail_on_subnet_address_without_port() { + Map subnetAddresses = + ImmutableMap.of("\"100.64.0.0/15\"", "cassandra.datacenter1.com"); + DefaultDriverContext context = context(subnetAddresses); + assertThatIllegalArgumentException() + .isThrownBy(() -> new SubnetAddressTranslator(context)) + .withMessage("Invalid address cassandra.datacenter1.com (expecting format host:port)"); + } + + @Test + public void should_fail_on_default_address_without_port() { + DefaultDriverContext context = context(ImmutableMap.of()); + when(context + .getConfig() + .getDefaultProfile() + .getString(ADDRESS_TRANSLATOR_DEFAULT_ADDRESS, null)) + .thenReturn("cassandra.com"); + assertThatIllegalArgumentException() + .isThrownBy(() -> new SubnetAddressTranslator(context)) + .withMessage("Invalid address cassandra.com (expecting format host:port)"); + } + + private static DefaultDriverContext context(Map subnetAddresses) { + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + when(profile.getStringMap(ADDRESS_TRANSLATOR_SUBNET_ADDRESSES)).thenReturn(subnetAddresses); + return MockedDriverContextFactory.defaultDriverContext(Optional.of(profile)); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetTest.java new file mode 100644 index 00000000000..f8ba8929e9e --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.addresstranslation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNoException; + +import java.net.UnknownHostException; +import org.junit.Test; + +public class SubnetTest { + @Test + public void should_parse_to_correct_ipv4_subnet() throws UnknownHostException { + Subnet subnet = Subnet.parse("100.64.0.0/15"); + assertThat(subnet.getSubnet()).containsExactly(100, 64, 0, 0); + assertThat(subnet.getNetworkMask()).containsExactly(255, 254, 0, 0); + assertThat(subnet.getUpper()).containsExactly(100, 65, 255, 255); + assertThat(subnet.getLower()).containsExactly(100, 64, 0, 0); + } + + @Test + public void should_parse_to_correct_ipv6_subnet() throws UnknownHostException { + Subnet subnet = Subnet.parse("2001:db8:85a3::8a2e:370:0/111"); + assertThat(subnet.getSubnet()) + .containsExactly(32, 1, 13, 184, 133, 163, 0, 0, 0, 0, 138, 46, 3, 112, 0, 0); + assertThat(subnet.getNetworkMask()) + .containsExactly( + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 254, 0, 0); + assertThat(subnet.getUpper()) + .containsExactly(32, 1, 13, 184, 133, 163, 0, 0, 0, 0, 138, 46, 3, 113, 255, 255); + assertThat(subnet.getLower()) + .containsExactly(32, 1, 13, 184, 133, 163, 0, 0, 0, 0, 138, 46, 3, 112, 0, 0); + } + + @Test + public void should_parse_to_correct_ipv6_subnet_ipv4_convertible() throws UnknownHostException { + Subnet subnet = Subnet.parse("::ffff:6440:0/111"); + assertThat(subnet.getSubnet()) + .containsExactly(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 100, 64, 0, 0); + assertThat(subnet.getNetworkMask()) + .containsExactly( + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 254, 0, 0); + assertThat(subnet.getUpper()) + .containsExactly(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 100, 65, 255, 255); + assertThat(subnet.getLower()) + .containsExactly(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 100, 64, 0, 0); + } + + @Test + public void should_fail_on_invalid_cidr_format() { + assertThatIllegalArgumentException() + .isThrownBy(() -> Subnet.parse("invalid")) + .withMessage("Invalid subnet: invalid"); + } + + @Test + public void should_parse_bounding_prefix_lengths_correctly() { + assertThatNoException().isThrownBy(() -> Subnet.parse("0.0.0.0/0")); + assertThatNoException().isThrownBy(() -> Subnet.parse("100.64.0.0/32")); + } + + @Test + public void should_fail_on_invalid_prefix_length() { + assertThatIllegalArgumentException() + .isThrownBy(() -> Subnet.parse("100.64.0.0/-1")) + .withMessage("Prefix length -1 must be within [0; 32]"); + assertThatIllegalArgumentException() + .isThrownBy(() -> Subnet.parse("100.64.0.0/33")) + .withMessage("Prefix length 33 must be within [0; 32]"); + } + + @Test + public void should_fail_on_not_prefix_block_subnet_ipv4() { + assertThatIllegalArgumentException() + .isThrownBy(() -> Subnet.parse("100.65.0.0/15")) + .withMessage("Subnet 100.65.0.0/15 must be represented as a network prefix block"); + } + + @Test + public void should_fail_on_not_prefix_block_subnet_ipv6() { + assertThatIllegalArgumentException() + .isThrownBy(() -> Subnet.parse("::ffff:6441:0/111")) + .withMessage("Subnet ::ffff:6441:0/111 must be represented as a network prefix block"); + } + + @Test + public void should_return_true_on_containing_address() throws UnknownHostException { + Subnet subnet = Subnet.parse("100.64.0.0/15"); + assertThat(subnet.contains(new byte[] {100, 64, 0, 0})).isTrue(); + assertThat(subnet.contains(new byte[] {100, 65, (byte) 255, (byte) 255})).isTrue(); + assertThat(subnet.contains(new byte[] {100, 65, 100, 100})).isTrue(); + } + + @Test + public void should_return_false_on_not_containing_address() throws UnknownHostException { + Subnet subnet = Subnet.parse("100.64.0.0/15"); + assertThat(subnet.contains(new byte[] {100, 63, (byte) 255, (byte) 255})).isFalse(); + assertThat(subnet.contains(new byte[] {100, 66, 0, 0})).isFalse(); + // IPv6 cannot be contained by IPv4 subnet. + assertThat(subnet.contains(new byte[16])).isFalse(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/config/MockOptions.java b/core/src/test/java/com/datastax/oss/driver/internal/core/config/MockOptions.java index 25c1e8b26fd..cee57abbfdf 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/config/MockOptions.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/config/MockOptions.java @@ -24,6 +24,7 @@ public enum MockOptions implements DriverOption { INT1("int1"), INT2("int2"), AUTH_PROVIDER("auth_provider"), + SUBNET_ADDRESSES("subnet_addresses"), ; private final String path; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfigTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfigTest.java index 16ccb73da9f..4a78c3ccb03 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfigTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfigTest.java @@ -101,7 +101,6 @@ public void should_fetch_string_map() { parse( "int1 = 42 \n auth_provider { auth_thing_one= one \n auth_thing_two = two \n auth_thing_three = three}"); DriverExecutionProfile base = config.getDefaultProfile(); - base.getStringMap(MockOptions.AUTH_PROVIDER); Map map = base.getStringMap(MockOptions.AUTH_PROVIDER); assertThat(map.entrySet().size()).isEqualTo(3); assertThat(map.get("auth_thing_one")).isEqualTo("one"); @@ -109,6 +108,19 @@ public void should_fetch_string_map() { assertThat(map.get("auth_thing_three")).isEqualTo("three"); } + @Test + public void should_fetch_string_map_with_forward_slash_in_keys() { + TypesafeDriverConfig config = + parse( + "subnet_addresses { 100.64.0.0/15 = \"cassandra.datacenter1.com:9042\" \n \"100.66.0.0/15\" = \"cassandra.datacenter2.com\" \n \"::ffff:6440:0/111\" = \"cassandra.datacenter3.com:19042\" }"); + DriverExecutionProfile base = config.getDefaultProfile(); + Map map = base.getStringMap(MockOptions.SUBNET_ADDRESSES); + assertThat(map.entrySet().size()).isEqualTo(3); + assertThat(map.get("100.64.0.\"0/15\"")).isEqualTo("cassandra.datacenter1.com:9042"); + assertThat(map.get("\"100.66.0.0/15\"")).isEqualTo("cassandra.datacenter2.com"); + assertThat(map.get("\"::ffff:6440:0/111\"")).isEqualTo("cassandra.datacenter3.com:19042"); + } + @Test public void should_create_derived_profile_with_string_map() { TypesafeDriverConfig config = parse("int1 = 42"); diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index 84efb4a796c..5b2536feb18 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -118,6 +118,55 @@ datastax-java-driver.advanced.address-translator.class = com.mycompany.MyAddress Note: the contact points provided while creating the `CqlSession` are not translated, only addresses retrieved from or sent by Cassandra nodes are. +### Fixed proxy hostname + +If your client applications access Cassandra through some kind of proxy (eg. with AWS PrivateLink when all Cassandra +nodes are exposed via one hostname pointing to AWS Endpoint), you can configure driver with +`FixedHostNameAddressTranslator` to always translate all node addresses to that same proxy hostname, no matter what IP +address a node has but still using its native transport port. + +To use it, specify the following in the [configuration](../configuration): + +``` +datastax-java-driver.advanced.address-translator.class = FixedHostNameAddressTranslator +advertised-hostname = proxyhostname +``` + +### Fixed proxy hostname per subnet + +When running Cassandra in a private network and accessing it from outside of that private network via some kind of +proxy, we have an option to use `FixedHostNameAddressTranslator`. But for multi-datacenter Cassandra deployments, we +want to have more control over routing queries to a specific datacenter (eg. for optimizing latencies), which requires +setting up a separate proxy per datacenter. + +Normally, each Cassandra datacenter nodes are deployed to a different subnet to support internode communications in the +cluster and avoid IP address collisions. So when Cassandra broadcasts its nodes IP addresses, we can determine which +datacenter that node belongs to by checking its IP address against the given datacenter subnet. + +For such scenarios you can use `SubnetAddressTranslator` to translate node IPs to the datacenter proxy address +associated with it. + +To use it, specify the following in the [configuration](../configuration): +``` +datastax-java-driver.advanced.address-translator { + class = SubnetAddressTranslator + subnet-addresses { + "100.64.0.0/15" = "cassandra.datacenter1.com:9042" + "100.66.0.0/15" = "cassandra.datacenter2.com:9042" + # IPv6 example: + # "::ffff:6440:0/111" = "cassandra.datacenter1.com:9042" + # "::ffff:6442:0/111" = "cassandra.datacenter2.com:9042" + } + # Optional. When configured, addresses not matching the configured subnets are translated to this address. + default-address = "cassandra.datacenter1.com:9042" + # Whether to resolve the addresses once on initialization (if true) or on each node (re-)connection (if false). + # If not configured, defaults to false. + resolve-addresses = false +} +``` + +Such setup is common for running Cassandra on Kubernetes with [k8ssandra](https://docs.k8ssandra.io/). + ### EC2 multi-region If you deploy both Cassandra and client applications on Amazon EC2, and your cluster spans multiple regions, you'll have From 29d3531202895fb9866bdd72720202c78a7eaa9b Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Tue, 20 May 2025 19:06:14 -0400 Subject: [PATCH 57/77] Fix revapi surious complaints about optional dependencies patch by Abe Ratnofsky; reviewed by Bret McGuire for CASSJAVA-102 --- Jenkinsfile-datastax | 2 +- core/revapi.json | 25 +++++++++++++++++++-- mapper-runtime/revapi.json | 6 ++--- pom.xml | 45 ++++++++++++++++++++++++-------------- query-builder/revapi.json | 9 ++++---- test-infra/revapi.json | 4 +--- 6 files changed, 61 insertions(+), 30 deletions(-) diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index af1aab6e0f4..73b977bdf9f 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -27,7 +27,7 @@ def initializeEnvironment() { env.GITHUB_BRANCH_URL = "${GITHUB_PROJECT_URL}/tree/${env.BRANCH_NAME}" env.GITHUB_COMMIT_URL = "${GITHUB_PROJECT_URL}/commit/${env.GIT_COMMIT}" - env.MAVEN_HOME = "${env.HOME}/.mvn/apache-maven-3.6.3" + env.MAVEN_HOME = "${env.HOME}/.mvn/apache-maven-3.8.8" env.PATH = "${env.MAVEN_HOME}/bin:${env.PATH}" /* diff --git a/core/revapi.json b/core/revapi.json index 5aa46a3ccad..f39c7d4a7c0 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -1,5 +1,3 @@ -// Configures Revapi (https://revapi.org/getting-started.html) to check API compatibility between -// successive driver versions. { "revapi": { "java": { @@ -7386,6 +7384,29 @@ "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.class.nonPublicPartOfAPI", + "old": "class com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverExecutionProfile.Base", + "justification": "CASSJAVA-102: Fix spurious complaints about optional dependencies" + }, + { + "code": "java.class.nonPublicPartOfAPI", + "old": "class com.fasterxml.jackson.databind.type.TypeParser.MyTokenizer", + "justification": "CASSJAVA-102: Fix spurious complaints about optional dependencies" + }, + { + "code": "java.class.nonPublicPartOfAPI", + "old": "class org.apache.tinkerpop.shaded.jackson.databind.type.TypeParser.MyTokenizer", + "justification": "CASSJAVA-102: Fix spurious complaints about optional dependencies" + }, + { + "code": "java.class.externalClassExposedInAPI", + "justification": "CASSJAVA-102: Migrate revapi config into dedicated config files, ported from pom.xml" + }, + { + "code": "java.method.varargOverloadsOnlyDifferInVarargParameter", + "justification": "CASSJAVA-102: Migrate revapi config into dedicated config files, ported from pom.xml" } ] } diff --git a/mapper-runtime/revapi.json b/mapper-runtime/revapi.json index 18d26a7f7e9..3dc2ea21671 100644 --- a/mapper-runtime/revapi.json +++ b/mapper-runtime/revapi.json @@ -1,5 +1,3 @@ -// Configures Revapi (https://revapi.org/getting-started.html) to check API compatibility between -// successive driver versions. { "revapi": { "java": { @@ -11,7 +9,7 @@ "com\\.datastax\\.(oss|dse)\\.driver\\.internal(\\..+)?", "com\\.datastax\\.oss\\.driver\\.shaded(\\..+)?", "com\\.datastax\\.oss\\.simulacron(\\..+)?", - // Don't re-check sibling modules that this module depends on + "// Don't re-check sibling modules that this module depends on", "com\\.datastax\\.(oss|dse)\\.driver\\.api\\.core(\\..+)?", "com\\.datastax\\.(oss|dse)\\.driver\\.api\\.querybuilder(\\..+)?" ] @@ -22,7 +20,7 @@ { "regex": true, "code": "java.annotation.attributeValueChanged", - "old": "@interface com\.datastax\.oss\.driver\.api\.mapper\.annotations\..*", + "old": "@interface com\\.datastax\\.oss\\.driver\\.api\\.mapper\\.annotations\\..*", "annotationType": "java.lang.annotation.Retention", "attribute": "value", "oldValue": "java.lang.annotation.RetentionPolicy.CLASS", diff --git a/pom.xml b/pom.xml index e5cfb58f94d..2cfeb65e757 100644 --- a/pom.xml +++ b/pom.xml @@ -561,28 +561,23 @@ org.revapi revapi-maven-plugin - 0.10.5 + 0.15.1 false \d+\.\d+\.\d+ - - - - - java.class.externalClassExposedInAPI - - - ${project.groupId}:${project.artifactId}:RELEASE + + revapi.json + org.revapi revapi-java - 0.22.1 + 0.28.4 @@ -596,9 +591,33 @@ flatten-maven-plugin 1.2.1 + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + maven-enforcer-plugin + + + enforce-maven + + enforce + + + + + + [3.8.1,) + + + + + + maven-compiler-plugin @@ -901,12 +920,6 @@ limitations under the License.]]> check - - - - revapi.json - - diff --git a/query-builder/revapi.json b/query-builder/revapi.json index c4d8aa27212..ed97379332c 100644 --- a/query-builder/revapi.json +++ b/query-builder/revapi.json @@ -1,5 +1,3 @@ -// Configures Revapi (https://revapi.org/getting-started.html) to check API compatibility between -// successive driver versions. { "revapi": { "java": { @@ -11,7 +9,7 @@ "com\\.datastax\\.(oss|dse)\\.driver\\.internal(\\..+)?", "com\\.datastax\\.oss\\.driver\\.shaded(\\..+)?", "org\\.assertj(\\..+)?", - // Don't re-check sibling modules that this module depends on + "// Don't re-check sibling modules that this module depends on", "com\\.datastax\\.(oss|dse)\\.driver\\.api\\.core(\\..+)?" ] } @@ -2782,8 +2780,11 @@ "code": "java.method.addedToInterface", "new": "method com.datastax.oss.driver.api.querybuilder.select.Select com.datastax.oss.driver.api.querybuilder.select.Select::orderByAnnOf(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector)", "justification": "JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder" + }, + { + "code": "java.method.varargOverloadsOnlyDifferInVarargParameter", + "justification": "CASSJAVA-102: Suppress newly-supported varargs check" } ] } } - diff --git a/test-infra/revapi.json b/test-infra/revapi.json index c75a98cb4af..293d9f4d142 100644 --- a/test-infra/revapi.json +++ b/test-infra/revapi.json @@ -1,5 +1,3 @@ -// Configures Revapi (https://revapi.org/getting-started.html) to check API compatibility between -// successive driver versions. { "revapi": { "java": { @@ -12,7 +10,7 @@ "com\\.datastax\\.oss\\.driver\\.shaded(\\..+)?", "com\\.datastax\\.oss\\.simulacron(\\..+)?", "org\\.assertj(\\..+)?", - // Don't re-check sibling modules that this module depends on + "// Don't re-check sibling modules that this module depends on", "com\\.datastax\\.(oss|dse)\\.driver\\.api\\.core(\\..+)?" ] } From f49e19b8c7e3bff6e5e4e8003484427c95bde027 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 7 Jul 2025 10:05:15 -0500 Subject: [PATCH 58/77] ninja-fix: updating OS label in Jenkinsfile after upgrade to Focal for runner --- Jenkinsfile-datastax | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index 73b977bdf9f..602f33101ca 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -402,7 +402,7 @@ pipeline { } environment { - OS_VERSION = 'ubuntu/bionic64/java-driver' + OS_VERSION = 'ubuntu/focal64/java-driver' JABBA_SHELL = '/usr/lib/jabba/jabba.sh' CCM_ENVIRONMENT_SHELL = '/usr/local/bin/ccm_environment.sh' SERIAL_ITS_ARGUMENT = "-DskipSerialITs=${params.SKIP_SERIAL_ITS}" From 17ebe6092e2877d8c524e07489c4c3d005cfeea5 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 14 Jul 2025 15:09:25 -0700 Subject: [PATCH 59/77] ninja-fix: openjdk@1.17.0 instead of openjdk@17 for ASF CI patch by Jane He; reviewed by Bret McGuire and Alexandre Dutra --- Jenkinsfile-asf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf index 24800ba9051..4b5041903c1 100644 --- a/Jenkinsfile-asf +++ b/Jenkinsfile-asf @@ -35,7 +35,7 @@ pipeline { axes { axis { name 'TEST_JAVA_VERSION' - values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@17', 'openjdk@1.21.0' + values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@1.17.0', 'openjdk@1.21.0' } axis { name 'SERVER_VERSION' From ddd6f03d7107df03e6e96e9fe37f434e4c742a9d Mon Sep 17 00:00:00 2001 From: Jason Koch Date: Mon, 17 Mar 2025 10:17:06 -0700 Subject: [PATCH 60/77] Remove unnecessary locking in DefaultNettyOptions This value is initialized at constructor time and marked final, so it can never change. There is no need to have access to this reference synchronized. Patch by Jason Koch; reviewed by Alexandre Dutra, Andy Tolbert and Jane He --- .../oss/driver/internal/core/context/DefaultNettyOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultNettyOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultNettyOptions.java index c5d3b3670f0..763a71f8b12 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultNettyOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultNettyOptions.java @@ -200,7 +200,7 @@ public Future onClose() { } @Override - public synchronized Timer getTimer() { + public Timer getTimer() { return timer; } } From f32069bd8abfae75f451ff4f47c44c1cca8dbd1e Mon Sep 17 00:00:00 2001 From: Michael Karsten Date: Fri, 21 Mar 2025 12:09:31 -0700 Subject: [PATCH 61/77] CASSJAVA-89 fix: support schema options that changed in Cassandra 5.0 Patch by Michael Karsten; reviewed by Abe Ratnofsky and Andy Tolbert for CASSJAVA-89 --- .../querybuilder/RelationOptionsIT.java | 131 ++++++++++++++++++ .../querybuilder/schema/RelationOptions.java | 126 ++++++++++++++--- .../schema/CreateDseTableTest.java | 65 +++++++++ .../querybuilder/schema/CreateTableTest.java | 55 ++++++++ 4 files changed, 355 insertions(+), 22 deletions(-) create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/querybuilder/RelationOptionsIT.java diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/querybuilder/RelationOptionsIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/querybuilder/RelationOptionsIT.java new file mode 100644 index 00000000000..fc571ccf44d --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/querybuilder/RelationOptionsIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.querybuilder; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.querybuilder.SchemaBuilder; +import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.categories.ParallelizableTests; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; + +@Category(ParallelizableTests.class) +public class RelationOptionsIT { + + private CcmRule ccmRule = CcmRule.getInstance(); + + private SessionRule sessionRule = SessionRule.builder(ccmRule).build(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + + @Rule public TestName name = new TestName(); + + @Test + @BackendRequirement( + type = BackendType.CASSANDRA, + minInclusive = "3.0", + description = "CRC check chance was moved to top level table in Cassandra 3.0") + public void should_create_table_with_crc_check_chance() { + sessionRule + .session() + .execute( + SchemaBuilder.createTable(name.getMethodName()) + .withPartitionKey("id", DataTypes.INT) + .withColumn("name", DataTypes.TEXT) + .withColumn("age", DataTypes.INT) + .withCRCCheckChance(0.8) + .build()); + KeyspaceMetadata keyspaceMetadata = + sessionRule + .session() + .getMetadata() + .getKeyspace(sessionRule.keyspace()) + .orElseThrow(AssertionError::new); + String describeOutput = keyspaceMetadata.describeWithChildren(true).trim(); + + assertThat(describeOutput).contains("crc_check_chance = 0.8"); + } + + @Test + @BackendRequirement( + type = BackendType.CASSANDRA, + minInclusive = "5.0", + description = "chunk_length_kb was renamed to chunk_length_in_kb in Cassandra 5.0") + public void should_create_table_with_chunk_length_in_kb() { + sessionRule + .session() + .execute( + SchemaBuilder.createTable(name.getMethodName()) + .withPartitionKey("id", DataTypes.INT) + .withColumn("name", DataTypes.TEXT) + .withColumn("age", DataTypes.INT) + .withLZ4Compression(4096) + .build()); + KeyspaceMetadata keyspaceMetadata = + sessionRule + .session() + .getMetadata() + .getKeyspace(sessionRule.keyspace()) + .orElseThrow(AssertionError::new); + String describeOutput = keyspaceMetadata.describeWithChildren(true).trim(); + + assertThat(describeOutput).contains("'class':'org.apache.cassandra.io.compress.LZ4Compressor'"); + assertThat(describeOutput).contains("'chunk_length_in_kb':'4096'"); + } + + @Test + @BackendRequirement( + type = BackendType.CASSANDRA, + minInclusive = "3.0", + maxExclusive = "5.0", + description = + "Deprecated compression options should still work with Cassandra >= 3.0 & < 5.0") + public void should_create_table_with_deprecated_options() { + sessionRule + .session() + .execute( + SchemaBuilder.createTable(name.getMethodName()) + .withPartitionKey("id", DataTypes.INT) + .withColumn("name", DataTypes.TEXT) + .withColumn("age", DataTypes.INT) + .withLZ4Compression(4096, 0.8) + .build()); + KeyspaceMetadata keyspaceMetadata = + sessionRule + .session() + .getMetadata() + .getKeyspace(sessionRule.keyspace()) + .orElseThrow(AssertionError::new); + String describeOutput = keyspaceMetadata.describeWithChildren(true).trim(); + + assertThat(describeOutput).contains("'class':'org.apache.cassandra.io.compress.LZ4Compressor'"); + assertThat(describeOutput).contains("'chunk_length_in_kb':'4096'"); + assertThat(describeOutput).contains("crc_check_chance = 0.8"); + } +} diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/RelationOptions.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/RelationOptions.java index 022562def81..49b342acb7f 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/RelationOptions.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/RelationOptions.java @@ -58,6 +58,18 @@ default SelfT withCDC(boolean enabled) { return withOption("cdc", enabled); } + /** + * Defines the crc check chance. + * + *

Note that using this option with a version of Apache Cassandra less than 3.0 will raise a + * syntax error. + */ + @NonNull + @CheckReturnValue + default SelfT withCRCCheckChance(double crcCheckChance) { + return withOption("crc_check_chance", crcCheckChance); + } + /** * Defines the caching criteria. * @@ -97,22 +109,32 @@ default SelfT withCompaction(@NonNull CompactionStrategy compactionStrategy) } /** - * Configures compression using the LZ4 algorithm with the given chunk length and crc check - * chance. - * - * @see #withCompression(String, int, double) + * @deprecated This method only exists for backward compatibility. Will not work with Apache + * Cassandra 5.0 or later. Use {@link #withLZ4Compression(int)} instead. */ + @Deprecated @NonNull @CheckReturnValue default SelfT withLZ4Compression(int chunkLengthKB, double crcCheckChance) { return withCompression("LZ4Compressor", chunkLengthKB, crcCheckChance); } + /** + * Configures compression using the LZ4 algorithm with the given chunk length. + * + * @see #withCompression(String, int) + */ + @NonNull + @CheckReturnValue + default SelfT withLZ4Compression(int chunkLengthKB) { + return withCompression("LZ4Compressor", chunkLengthKB); + } + /** * Configures compression using the LZ4 algorithm using the default configuration (64kb - * chunk_length, and 1.0 crc_check_chance). + * chunk_length). * - * @see #withCompression(String, int, double) + * @see #withCompression(String, int) */ @NonNull @CheckReturnValue @@ -121,22 +143,57 @@ default SelfT withLZ4Compression() { } /** - * Configures compression using the Snappy algorithm with the given chunk length and crc check - * chance. + * Configures compression using the Zstd algorithm with the given chunk length. * - * @see #withCompression(String, int, double) + * @see #withCompression(String, int) */ @NonNull @CheckReturnValue + default SelfT withZstdCompression(int chunkLengthKB) { + return withCompression("ZstdCompressor", chunkLengthKB); + } + + /** + * Configures compression using the Zstd algorithm using the default configuration (64kb + * chunk_length). + * + * @see #withCompression(String, int) + */ + @NonNull + @CheckReturnValue + default SelfT withZstdCompression() { + return withCompression("ZstdCompressor"); + } + + /** + * @deprecated This method only exists for backward compatibility. Will not work with Apache + * Cassandra 5.0 or later due to removal of deprecated table properties (CASSANDRA-18742). Use + * {@link #withSnappyCompression(int)} instead. + */ + @Deprecated + @NonNull + @CheckReturnValue default SelfT withSnappyCompression(int chunkLengthKB, double crcCheckChance) { return withCompression("SnappyCompressor", chunkLengthKB, crcCheckChance); } + /** + * Configures compression using the Snappy algorithm with the given chunk length. + * + * @see #withCompression(String, int) + */ + @NonNull + @CheckReturnValue + default SelfT withSnappyCompression(int chunkLengthKB) { + return withCompression("SnappyCompressor", chunkLengthKB); + } + /** * Configures compression using the Snappy algorithm using the default configuration (64kb - * chunk_length, and 1.0 crc_check_chance). + * chunk_length). * - * @see #withCompression(String, int, double) + * @see #withCompression(String, int) */ @NonNull @CheckReturnValue @@ -145,22 +202,34 @@ default SelfT withSnappyCompression() { } /** - * Configures compression using the Deflate algorithm with the given chunk length and crc check - * chance. - * - * @see #withCompression(String, int, double) + * @deprecated This method only exists for backward compatibility. Will not work with Apache + * Cassandra 5.0 or later due to removal of deprecated table properties (CASSANDRA-18742). Use + * {@link #withDeflateCompression(int)} instead. */ + @Deprecated @NonNull @CheckReturnValue default SelfT withDeflateCompression(int chunkLengthKB, double crcCheckChance) { return withCompression("DeflateCompressor", chunkLengthKB, crcCheckChance); } + /** + * Configures compression using the Deflate algorithm with the given chunk length. + * + * @see #withCompression(String, int) + */ + @NonNull + @CheckReturnValue + default SelfT withDeflateCompression(int chunkLengthKB) { + return withCompression("DeflateCompressor", chunkLengthKB); + } + /** * Configures compression using the Deflate algorithm using the default configuration (64kb - * chunk_length, and 1.0 crc_check_chance). + * chunk_length). * - * @see #withCompression(String, int, double) + * @see #withCompression(String, int) */ @NonNull @CheckReturnValue @@ -170,13 +239,13 @@ default SelfT withDeflateCompression() { /** * Configures compression using the given algorithm using the default configuration (64kb - * chunk_length, and 1.0 crc_check_chance). + * chunk_length). * *

Unless specifying a custom compression algorithm implementation, it is recommended to use * {@link #withLZ4Compression()}, {@link #withSnappyCompression()}, or {@link * #withDeflateCompression()}. * - * @see #withCompression(String, int, double) + * @see #withCompression(String, int) */ @NonNull @CheckReturnValue @@ -185,7 +254,7 @@ default SelfT withCompression(@NonNull String compressionAlgorithmName) { } /** - * Configures compression using the given algorithm, chunk length and crc check chance. + * Configures compression using the given algorithm, chunk length. * *

Unless specifying a custom compression algorithm implementation, it is recommended to use * {@link #withLZ4Compression()}, {@link #withSnappyCompression()}, or {@link @@ -193,11 +262,24 @@ default SelfT withCompression(@NonNull String compressionAlgorithmName) { * * @param compressionAlgorithmName The class name of the compression algorithm. * @param chunkLengthKB The chunk length in KB of compression blocks. Defaults to 64. - * @param crcCheckChance The probability (0.0 to 1.0) that checksum will be checked on each read. - * Defaults to 1.0. */ @NonNull @CheckReturnValue + default SelfT withCompression(@NonNull String compressionAlgorithmName, int chunkLengthKB) { + return withOption( + "compression", + ImmutableMap.of("class", compressionAlgorithmName, "chunk_length_in_kb", chunkLengthKB)); + } + + /** + * @deprecated This method only exists for backward compatibility. Will not work with Apache + * Cassandra 5.0 or later due to removal of deprecated table properties (CASSANDRA-18742). Use + * {@link #withCompression(String, int)} instead. + */ + @NonNull + @CheckReturnValue + @Deprecated default SelfT withCompression( @NonNull String compressionAlgorithmName, int chunkLengthKB, double crcCheckChance) { return withOption( diff --git a/query-builder/src/test/java/com/datastax/dse/driver/api/querybuilder/schema/CreateDseTableTest.java b/query-builder/src/test/java/com/datastax/dse/driver/api/querybuilder/schema/CreateDseTableTest.java index 7fec9674628..d8ee1c4e380 100644 --- a/query-builder/src/test/java/com/datastax/dse/driver/api/querybuilder/schema/CreateDseTableTest.java +++ b/query-builder/src/test/java/com/datastax/dse/driver/api/querybuilder/schema/CreateDseTableTest.java @@ -195,6 +195,17 @@ public void should_generate_create_table_lz4_compression() { @Test public void should_generate_create_table_lz4_compression_options() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withLZ4Compression(1024)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'LZ4Compressor','chunk_length_in_kb':1024}"); + } + + @Test + public void should_generate_create_table_lz4_compression_options_crc() { assertThat( createDseTable("bar") .withPartitionKey("k", DataTypes.INT) @@ -204,6 +215,28 @@ public void should_generate_create_table_lz4_compression_options() { "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'LZ4Compressor','chunk_length_kb':1024,'crc_check_chance':0.5}"); } + @Test + public void should_generate_create_table_zstd_compression() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withZstdCompression()) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'ZstdCompressor'}"); + } + + @Test + public void should_generate_create_table_zstd_compression_options() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withZstdCompression(1024)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'ZstdCompressor','chunk_length_in_kb':1024}"); + } + @Test public void should_generate_create_table_snappy_compression() { assertThat( @@ -217,6 +250,17 @@ public void should_generate_create_table_snappy_compression() { @Test public void should_generate_create_table_snappy_compression_options() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withSnappyCompression(2048)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'SnappyCompressor','chunk_length_in_kb':2048}"); + } + + @Test + public void should_generate_create_table_snappy_compression_options_crc() { assertThat( createDseTable("bar") .withPartitionKey("k", DataTypes.INT) @@ -239,6 +283,17 @@ public void should_generate_create_table_deflate_compression() { @Test public void should_generate_create_table_deflate_compression_options() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withDeflateCompression(4096)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'DeflateCompressor','chunk_length_in_kb':4096}"); + } + + @Test + public void should_generate_create_table_deflate_compression_options_crc() { assertThat( createDseTable("bar") .withPartitionKey("k", DataTypes.INT) @@ -389,4 +444,14 @@ public void should_generate_create_table_with_named_edge() { + "FROM person(contributor) " + "TO soft((company_name,software_name),software_version)"); } + + @Test + public void should_generate_create_table_crc_check_chance() { + assertThat( + createDseTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withCRCCheckChance(0.8)) + .hasCql("CREATE TABLE bar (k int PRIMARY KEY,v text) WITH crc_check_chance=0.8"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java index 15cd12c75eb..31efc278472 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java @@ -199,6 +199,17 @@ public void should_generate_create_table_lz4_compression() { @Test public void should_generate_create_table_lz4_compression_options() { + assertThat( + createTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withLZ4Compression(1024)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'LZ4Compressor','chunk_length_in_kb':1024}"); + } + + @Test + public void should_generate_create_table_lz4_compression_options_crc() { assertThat( createTable("bar") .withPartitionKey("k", DataTypes.INT) @@ -208,6 +219,28 @@ public void should_generate_create_table_lz4_compression_options() { "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'LZ4Compressor','chunk_length_kb':1024,'crc_check_chance':0.5}"); } + @Test + public void should_generate_create_table_zstd_compression() { + assertThat( + createTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withZstdCompression()) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'ZstdCompressor'}"); + } + + @Test + public void should_generate_create_table_zstd_compression_options() { + assertThat( + createTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withZstdCompression(1024)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'ZstdCompressor','chunk_length_in_kb':1024}"); + } + @Test public void should_generate_create_table_snappy_compression() { assertThat( @@ -221,6 +254,17 @@ public void should_generate_create_table_snappy_compression() { @Test public void should_generate_create_table_snappy_compression_options() { + assertThat( + createTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withSnappyCompression(2048)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'SnappyCompressor','chunk_length_in_kb':2048}"); + } + + @Test + public void should_generate_create_table_snappy_compression_options_crc() { assertThat( createTable("bar") .withPartitionKey("k", DataTypes.INT) @@ -243,6 +287,17 @@ public void should_generate_create_table_deflate_compression() { @Test public void should_generate_create_table_deflate_compression_options() { + assertThat( + createTable("bar") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.TEXT) + .withDeflateCompression(4096)) + .hasCql( + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compression={'class':'DeflateCompressor','chunk_length_in_kb':4096}"); + } + + @Test + public void should_generate_create_table_deflate_compression_options_crc() { assertThat( createTable("bar") .withPartitionKey("k", DataTypes.INT) From 7e21eb20283f3781a0a748741b768d0adf0fc85b Mon Sep 17 00:00:00 2001 From: Jason Koch Date: Mon, 3 Feb 2025 13:46:32 -0800 Subject: [PATCH 62/77] Eliminate lock in ConcurrencyLimitingRequestThrottler Following from 6d3ba47 this changes the throttler to a complete lock-free implementation. Update the related comments and README now that it is lock-free. Patch by Jason Koch; reviewed by Alexandre Dutra and Andy Tolbert --- .../session/throttling/RequestThrottler.java | 8 +- .../ConcurrencyLimitingRequestThrottler.java | 180 ++++++++---------- manual/core/non_blocking/README.md | 14 +- 3 files changed, 90 insertions(+), 112 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java index 7e2b41ebbdb..73d347d533e 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java @@ -23,10 +23,10 @@ /** * Limits the number of concurrent requests executed by the driver. * - *

Usage in non-blocking applications: beware that all built-in implementations of this interface - * use locks for internal coordination, and do not qualify as lock-free, with the obvious exception - * of {@code PassThroughRequestThrottler}. If your application enforces strict lock-freedom, then - * request throttling should not be enabled. + *

Usage in non-blocking applications: beware that some implementations of this interface use + * locks for internal coordination, and do not qualify as lock-free. If your application enforces + * strict lock-freedom, then you should use the {@code PassThroughRequestThrottler} or the {@code + * ConcurrencyLimitingRequestThrottler}. */ public interface RequestThrottler extends Closeable { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java index ffe0ffe9650..8146c5b113a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java @@ -26,10 +26,9 @@ import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.util.ArrayDeque; import java.util.Deque; -import java.util.concurrent.locks.ReentrantLock; -import net.jcip.annotations.GuardedBy; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +60,12 @@ public class ConcurrencyLimitingRequestThrottler implements RequestThrottler { private final String logPrefix; private final int maxConcurrentRequests; private final int maxQueueSize; - - private final ReentrantLock lock = new ReentrantLock(); - - @GuardedBy("lock") - private int concurrentRequests; - - @GuardedBy("lock") - private final Deque queue = new ArrayDeque<>(); - - @GuardedBy("lock") - private boolean closed; + private final AtomicInteger concurrentRequests = new AtomicInteger(0); + // CLQ is not O(1) for size(), as it forces a full iteration of the queue. So, we track + // the size of the queue explicitly. + private final Deque queue = new ConcurrentLinkedDeque<>(); + private final AtomicInteger queueSize = new AtomicInteger(0); + private volatile boolean closed = false; public ConcurrencyLimitingRequestThrottler(DriverContext context) { this.logPrefix = context.getSessionName(); @@ -88,50 +82,62 @@ public ConcurrencyLimitingRequestThrottler(DriverContext context) { @Override public void register(@NonNull Throttled request) { - boolean notifyReadyRequired = false; + if (closed) { + LOG.trace("[{}] Rejecting request after shutdown", logPrefix); + fail(request, "The session is shutting down"); + return; + } - lock.lock(); - try { - if (closed) { - LOG.trace("[{}] Rejecting request after shutdown", logPrefix); - fail(request, "The session is shutting down"); - } else if (queue.isEmpty() && concurrentRequests < maxConcurrentRequests) { - // We have capacity for one more concurrent request + // Implementation note: Technically the "concurrent requests" or "queue size" + // could read transiently over the limit, but the queue itself will never grow + // beyond the limit since we always check for that condition and revert if + // over-limit. We do this instead of a CAS-loop to avoid the potential loop. + + // If no backlog exists AND we get capacity, we can execute immediately + if (queueSize.get() == 0) { + // Take a claim first, and then check if we are OK to proceed + int newConcurrent = concurrentRequests.incrementAndGet(); + if (newConcurrent <= maxConcurrentRequests) { LOG.trace("[{}] Starting newly registered request", logPrefix); - concurrentRequests += 1; - notifyReadyRequired = true; - } else if (queue.size() < maxQueueSize) { - LOG.trace("[{}] Enqueuing request", logPrefix); - queue.add(request); + request.onThrottleReady(false); + return; } else { - LOG.trace("[{}] Rejecting request because of full queue", logPrefix); - fail( - request, - String.format( - "The session has reached its maximum capacity " - + "(concurrent requests: %d, queue size: %d)", - maxConcurrentRequests, maxQueueSize)); + // We exceeded the limit, decrement the count and fall through to the queuing logic + concurrentRequests.decrementAndGet(); } - } finally { - lock.unlock(); } - // no need to hold the lock while allowing the task to progress - if (notifyReadyRequired) { - request.onThrottleReady(false); + // If we have a backlog, or we failed to claim capacity, try to enqueue + int newQueueSize = queueSize.incrementAndGet(); + if (newQueueSize <= maxQueueSize) { + LOG.trace("[{}] Enqueuing request", logPrefix); + queue.offer(request); + + // Double-check that we were still supposed to be enqueued; it is possible + // that the session was closed while we were enqueuing, it's also possible + // that it is right now removing the request, so we need to check both + if (closed) { + if (queue.remove(request)) { + queueSize.decrementAndGet(); + LOG.trace("[{}] Rejecting late request after shutdown", logPrefix); + fail(request, "The session is shutting down"); + } + } + } else { + LOG.trace("[{}] Rejecting request because of full queue", logPrefix); + queueSize.decrementAndGet(); + fail( + request, + String.format( + "The session has reached its maximum capacity " + + "(concurrent requests: %d, queue size: %d)", + maxConcurrentRequests, maxQueueSize)); } } @Override public void signalSuccess(@NonNull Throttled request) { - Throttled nextRequest = null; - lock.lock(); - try { - nextRequest = onRequestDoneAndDequeNext(); - } finally { - lock.unlock(); - } - + Throttled nextRequest = onRequestDoneAndDequeNext(); if (nextRequest != null) { nextRequest.onThrottleReady(true); } @@ -145,17 +151,13 @@ public void signalError(@NonNull Throttled request, @NonNull Throwable error) { @Override public void signalTimeout(@NonNull Throttled request) { Throttled nextRequest = null; - lock.lock(); - try { - if (!closed) { - if (queue.remove(request)) { // The request timed out before it was active - LOG.trace("[{}] Removing timed out request from the queue", logPrefix); - } else { - nextRequest = onRequestDoneAndDequeNext(); - } + if (!closed) { + if (queue.remove(request)) { // The request timed out before it was active + queueSize.decrementAndGet(); + LOG.trace("[{}] Removing timed out request from the queue", logPrefix); + } else { + nextRequest = onRequestDoneAndDequeNext(); } - } finally { - lock.unlock(); } if (nextRequest != null) { @@ -166,17 +168,13 @@ public void signalTimeout(@NonNull Throttled request) { @Override public void signalCancel(@NonNull Throttled request) { Throttled nextRequest = null; - lock.lock(); - try { - if (!closed) { - if (queue.remove(request)) { // The request has been cancelled before it was active - LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); - } else { - nextRequest = onRequestDoneAndDequeNext(); - } + if (!closed) { + if (queue.remove(request)) { // The request has been cancelled before it was active + queueSize.decrementAndGet(); + LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); + } else { + nextRequest = onRequestDoneAndDequeNext(); } - } finally { - lock.unlock(); } if (nextRequest != null) { @@ -184,17 +182,16 @@ public void signalCancel(@NonNull Throttled request) { } } - @SuppressWarnings("GuardedBy") // this method is only called with the lock held @Nullable private Throttled onRequestDoneAndDequeNext() { - assert lock.isHeldByCurrentThread(); if (!closed) { - if (queue.isEmpty()) { - concurrentRequests -= 1; + Throttled nextRequest = queue.poll(); + if (nextRequest == null) { + concurrentRequests.decrementAndGet(); } else { + queueSize.decrementAndGet(); LOG.trace("[{}] Starting dequeued request", logPrefix); - // don't touch concurrentRequests since we finished one but started another - return queue.poll(); + return nextRequest; } } @@ -204,45 +201,28 @@ private Throttled onRequestDoneAndDequeNext() { @Override public void close() { - lock.lock(); - try { - closed = true; - LOG.debug("[{}] Rejecting {} queued requests after shutdown", logPrefix, queue.size()); - for (Throttled request : queue) { - fail(request, "The session is shutting down"); - } - } finally { - lock.unlock(); + closed = true; + + LOG.debug("[{}] Rejecting {} queued requests after shutdown", logPrefix, queueSize.get()); + Throttled request; + while ((request = queue.poll()) != null) { + queueSize.decrementAndGet(); + fail(request, "The session is shutting down"); } } public int getQueueSize() { - lock.lock(); - try { - return queue.size(); - } finally { - lock.unlock(); - } + return queueSize.get(); } @VisibleForTesting int getConcurrentRequests() { - lock.lock(); - try { - return concurrentRequests; - } finally { - lock.unlock(); - } + return concurrentRequests.get(); } @VisibleForTesting Deque getQueue() { - lock.lock(); - try { - return queue; - } finally { - lock.unlock(); - } + return queue; } private static void fail(Throttled request, String message) { diff --git a/manual/core/non_blocking/README.md b/manual/core/non_blocking/README.md index 7abe9d856a3..f320ffd13d2 100644 --- a/manual/core/non_blocking/README.md +++ b/manual/core/non_blocking/README.md @@ -152,15 +152,13 @@ should not be used if strict lock-freedom is enforced. [`SafeInitNodeStateListener`]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/metadata/SafeInitNodeStateListener.html -The same is valid for both built-in [request throttlers]: +The `RateLimitingRequestThrottler` is currently blocking. The `ConcurrencyLimitingRequestThrottler` +is lock-free. -* `ConcurrencyLimitingRequestThrottler` -* `RateLimitingRequestThrottler` - -See the section about [throttling](../throttling) for details about these components. Again, they -use locks internally, and depending on how many requests are being executed in parallel, the thread -contention on these locks can be high: in short, if your application enforces strict lock-freedom, -then these components should not be used. +See the section about [throttling](../throttling) for details about these components. Depending on +how many requests are being executed in parallel, the thread contention on these locks can be high: +in short, if your application enforces strict lock-freedom, then you should not use the +`RateLimitingRequestThrottler`. [request throttlers]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.html From 69eebb939c71dbb709099f7bf04b1a8b7e17012f Mon Sep 17 00:00:00 2001 From: Ivan Sopov Date: Mon, 28 Jul 2025 11:28:59 +0300 Subject: [PATCH 63/77] Change groupId in README patch by Ivan Sopov; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/2049 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b6e1cc337d8..0f6c2bb5a6f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ and Cassandra Query Language (CQL) v3. ## Getting the driver -The driver artifacts are published in Maven central, under the group id [com.datastax.oss]; there +The driver artifacts are published in Maven central, under the group id [org.apache.cassandra]; there are multiple modules, all prefixed with `java-driver-`. ```xml @@ -48,7 +48,7 @@ dependency if you plan to use it. Refer to each module's manual for more details ([core](manual/core/), [query builder](manual/query_builder/), [mapper](manual/mapper)). -[com.datastax.oss]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.datastax.oss%22 +[org.apache.cassandra]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.cassandra%22 ## Compatibility From 05e6717253d1c6ae0c5a9ce20fcf5ab448d32ec6 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 12 Jul 2024 09:37:06 +0800 Subject: [PATCH 64/77] manual: correct the codeblock directive patch by Kefu Chai; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1940 --- manual/mapper/daos/getentity/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/mapper/daos/getentity/README.md b/manual/mapper/daos/getentity/README.md index abb7cb076c8..de9a530b558 100644 --- a/manual/mapper/daos/getentity/README.md +++ b/manual/mapper/daos/getentity/README.md @@ -108,7 +108,7 @@ The method can return: * a single entity instance. If the argument is a result set type, the generated code will extract the first row and convert it, or return `null` if the result set is empty. - ````java + ```java @GetEntity Product asProduct(Row row); From ff2d7f26c63e7a3ed3a74906771b246949112414 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 14 Apr 2025 16:40:53 +0200 Subject: [PATCH 65/77] CASSJAVA-92: Local DC provided for nodetool clientstats patch by Lukasz Antoniak; reviewed by Bret McGuire and Abe Ratnofsky for CASSJAVA-92 --- .../loadbalancing/LoadBalancingPolicy.java | 7 + .../core/context/DefaultDriverContext.java | 14 +- .../core/context/StartupOptionsBuilder.java | 28 ++++ .../BasicLoadBalancingPolicy.java | 31 ++++- .../DefaultLoadBalancingPolicy.java | 10 ++ .../helper/OptionalLocalDcHelper.java | 31 +++-- .../FixedHostNameAddressTranslatorTest.java | 3 +- .../SubnetAddressTranslatorTest.java | 3 +- .../context/DefaultDriverContextTest.java | 2 +- .../context/MockedDriverContextFactory.java | 124 +++++++++++++++--- .../context/StartupOptionsBuilderTest.java | 45 ++++++- 11 files changed, 252 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/loadbalancing/LoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/api/core/loadbalancing/LoadBalancingPolicy.java index d890ae6c100..de0d9db4ebd 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/loadbalancing/LoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/loadbalancing/LoadBalancingPolicy.java @@ -24,6 +24,7 @@ import com.datastax.oss.driver.api.core.tracker.RequestTracker; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -76,6 +77,12 @@ default Optional getRequestTracker() { */ void init(@NonNull Map nodes, @NonNull DistanceReporter distanceReporter); + /** Returns map containing details that impact C* node connectivity. */ + @NonNull + default Map getStartupConfiguration() { + return Collections.emptyMap(); + } + /** * Returns the coordinators to use for a new query. * diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java index a24b632f640..0d7db27dfbe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java @@ -216,8 +216,8 @@ public class DefaultDriverContext implements InternalDriverContext { new LazyReference<>("metricIdGenerator", this::buildMetricIdGenerator, cycleDetector); private final LazyReference requestThrottlerRef = new LazyReference<>("requestThrottler", this::buildRequestThrottler, cycleDetector); - private final LazyReference> startupOptionsRef = - new LazyReference<>("startupOptions", this::buildStartupOptions, cycleDetector); + private final LazyReference startupOptionsRef = + new LazyReference<>("startupOptionsFactory", this::buildStartupOptionsFactory, cycleDetector); private final LazyReference nodeStateListenerRef; private final LazyReference schemaChangeListenerRef; private final LazyReference requestTrackerRef; @@ -335,16 +335,15 @@ public DefaultDriverContext( } /** - * Builds a map of options to send in a Startup message. + * Returns builder of options to send in a Startup message. * * @see #getStartupOptions() */ - protected Map buildStartupOptions() { + protected StartupOptionsBuilder buildStartupOptionsFactory() { return new StartupOptionsBuilder(this) .withClientId(startupClientId) .withApplicationName(startupApplicationName) - .withApplicationVersion(startupApplicationVersion) - .build(); + .withApplicationVersion(startupApplicationVersion); } protected Map buildLoadBalancingPolicies() { @@ -1013,7 +1012,8 @@ public ProtocolVersion getProtocolVersion() { @NonNull @Override public Map getStartupOptions() { - return startupOptionsRef.get(); + // startup options are calculated dynamically and may vary per connection + return startupOptionsRef.get().build(); } protected RequestLogFormatter buildRequestLogFormatter() { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilder.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilder.java index 684d6b01b9c..89a9266b3ac 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilder.java @@ -19,24 +19,34 @@ import com.datastax.dse.driver.api.core.config.DseDriverOption; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; +import com.fasterxml.jackson.databind.ObjectMapper; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.Map; +import java.util.Optional; import java.util.UUID; import net.jcip.annotations.Immutable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Immutable public class StartupOptionsBuilder { public static final String DRIVER_NAME_KEY = "DRIVER_NAME"; public static final String DRIVER_VERSION_KEY = "DRIVER_VERSION"; + public static final String DRIVER_BAGGAGE = "DRIVER_BAGGAGE"; public static final String APPLICATION_NAME_KEY = "APPLICATION_NAME"; public static final String APPLICATION_VERSION_KEY = "APPLICATION_VERSION"; public static final String CLIENT_ID_KEY = "CLIENT_ID"; + private static final Logger LOG = LoggerFactory.getLogger(StartupOptionsBuilder.class); + private static final ObjectMapper mapper = new ObjectMapper(); + protected final InternalDriverContext context; private UUID clientId; private String applicationName; @@ -119,6 +129,7 @@ public Map build() { if (applicationVersion != null) { builder.put(APPLICATION_VERSION_KEY, applicationVersion); } + driverBaggage().ifPresent(s -> builder.put(DRIVER_BAGGAGE, s)); return builder.build(); } @@ -142,4 +153,21 @@ protected String getDriverName() { protected String getDriverVersion() { return Session.OSS_DRIVER_COORDINATES.getVersion().toString(); } + + private Optional driverBaggage() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (Map.Entry entry : + context.getLoadBalancingPolicies().entrySet()) { + Map config = entry.getValue().getStartupConfiguration(); + if (!config.isEmpty()) { + builder.put(entry.getKey(), config); + } + } + try { + return Optional.of(mapper.writeValueAsString(builder.build())); + } catch (Exception e) { + LOG.warn("Failed to construct startup driver baggage", e); + return Optional.empty(); + } + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/BasicLoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/BasicLoadBalancingPolicy.java index 587ef4183bd..a02a5eb3148 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/BasicLoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/BasicLoadBalancingPolicy.java @@ -45,6 +45,7 @@ import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.base.Predicates; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import com.datastax.oss.driver.shaded.guava.common.collect.Sets; import edu.umd.cs.findbugs.annotations.NonNull; @@ -155,10 +156,38 @@ public BasicLoadBalancingPolicy(@NonNull DriverContext context, @NonNull String * Before initialization, this method always returns null. */ @Nullable - protected String getLocalDatacenter() { + public String getLocalDatacenter() { return localDc; } + @NonNull + @Override + public Map getStartupConfiguration() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + if (localDc != null) { + builder.put("localDc", localDc); + } else { + // Local data center may not be discovered prior to connection pool initialization. + // In such scenario, return configured local data center name. + // Note that when using DC inferring load balancing policy, startup configuration + // may not show local DC name, because it will be discovered only once control connection + // is established and datacenter of contact points known. + Optional configuredDc = + new OptionalLocalDcHelper(context, profile, logPrefix).configuredLocalDc(); + configuredDc.ifPresent(d -> builder.put("localDc", d)); + } + if (!preferredRemoteDcs.isEmpty()) { + builder.put("preferredRemoteDcs", preferredRemoteDcs); + } + if (allowDcFailoverForLocalCl) { + builder.put("allowDcFailoverForLocalCl", allowDcFailoverForLocalCl); + } + if (maxNodesPerRemoteDc > 0) { + builder.put("maxNodesPerRemoteDc", maxNodesPerRemoteDc); + } + return ImmutableMap.of(BasicLoadBalancingPolicy.class.getSimpleName(), builder.build()); + } + /** @return The nodes currently considered as live. */ protected NodeSet getLiveNodes() { return liveNodes; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java index 0f03cbb3643..9c31b606f18 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java @@ -34,6 +34,7 @@ import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.MapMaker; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -350,4 +351,13 @@ private boolean hasSufficientResponses(long now) { return this.oldest - threshold >= 0; } } + + @NonNull + @Override + public Map getStartupConfiguration() { + Map parent = super.getStartupConfiguration(); + return ImmutableMap.of( + DefaultLoadBalancingPolicy.class.getSimpleName(), + parent.get(BasicLoadBalancingPolicy.class.getSimpleName())); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java index d470f96c42c..c6143f3fa16 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java @@ -65,20 +65,14 @@ public OptionalLocalDcHelper( @Override @NonNull public Optional discoverLocalDc(@NonNull Map nodes) { - String localDc = context.getLocalDatacenter(profile.getName()); - if (localDc != null) { - LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); - checkLocalDatacenterCompatibility(localDc, context.getMetadataManager().getContactPoints()); - return Optional.of(localDc); - } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { - localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); - LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); - checkLocalDatacenterCompatibility(localDc, context.getMetadataManager().getContactPoints()); - return Optional.of(localDc); + Optional localDc = configuredLocalDc(); + if (localDc.isPresent()) { + checkLocalDatacenterCompatibility( + localDc.get(), context.getMetadataManager().getContactPoints()); } else { LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix); - return Optional.empty(); } + return localDc; } /** @@ -138,4 +132,19 @@ protected String formatDcs(Iterable nodes) { } return String.join(", ", new TreeSet<>(l)); } + + /** @return Local data center set programmatically or from configuration file. */ + @NonNull + public Optional configuredLocalDc() { + String localDc = context.getLocalDatacenter(profile.getName()); + if (localDc != null) { + LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); + return Optional.of(localDc); + } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { + localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); + LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); + return Optional.of(localDc); + } + return Optional.empty(); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java index 92800998056..3bb9c4bc291 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java @@ -26,7 +26,6 @@ import com.datastax.oss.driver.internal.core.context.DefaultDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; import java.net.InetSocketAddress; -import java.util.Optional; import org.junit.Test; public class FixedHostNameAddressTranslatorTest { @@ -36,7 +35,7 @@ public void should_translate_address() { DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); when(defaultProfile.getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME)).thenReturn("myaddress"); DefaultDriverContext defaultDriverContext = - MockedDriverContextFactory.defaultDriverContext(Optional.of(defaultProfile)); + MockedDriverContextFactory.defaultDriverContext(defaultProfile); FixedHostNameAddressTranslator translator = new FixedHostNameAddressTranslator(defaultDriverContext); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java index 2aa6ae75bc2..420170654dc 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/SubnetAddressTranslatorTest.java @@ -30,7 +30,6 @@ import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import java.net.InetSocketAddress; import java.util.Map; -import java.util.Optional; import org.junit.Test; @SuppressWarnings("resource") @@ -148,6 +147,6 @@ public void should_fail_on_default_address_without_port() { private static DefaultDriverContext context(Map subnetAddresses) { DriverExecutionProfile profile = mock(DriverExecutionProfile.class); when(profile.getStringMap(ADDRESS_TRANSLATOR_SUBNET_ADDRESSES)).thenReturn(subnetAddresses); - return MockedDriverContextFactory.defaultDriverContext(Optional.of(profile)); + return MockedDriverContextFactory.defaultDriverContext(profile); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContextTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContextTest.java index baf101508d4..6d4585cb4d7 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContextTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContextTest.java @@ -42,7 +42,7 @@ private DefaultDriverContext buildMockedContext(Optional compressionOpti DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); when(defaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION, "none")) .thenReturn(compressionOption.orElse("none")); - return MockedDriverContextFactory.defaultDriverContext(Optional.of(defaultProfile)); + return MockedDriverContextFactory.defaultDriverContext(defaultProfile); } private void doCreateCompressorTest(Optional configVal, Class expectedClz) { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/context/MockedDriverContextFactory.java b/core/src/test/java/com/datastax/oss/driver/internal/core/context/MockedDriverContextFactory.java index 06817326844..a8b25193f54 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/context/MockedDriverContextFactory.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/context/MockedDriverContextFactory.java @@ -24,44 +24,45 @@ import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; +import com.datastax.oss.driver.api.core.loadbalancing.NodeDistanceEvaluator; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeStateListener; import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; import com.datastax.oss.driver.api.core.tracker.RequestTracker; +import com.datastax.oss.driver.internal.core.ConsistencyLevelRegistry; +import com.datastax.oss.driver.internal.core.loadbalancing.DefaultLoadBalancingPolicy; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.Maps; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.UUID; public class MockedDriverContextFactory { public static DefaultDriverContext defaultDriverContext() { - return defaultDriverContext(Optional.empty()); + return defaultDriverContext(MockedDriverContextFactory.defaultProfile("datacenter1")); } public static DefaultDriverContext defaultDriverContext( - Optional profileOption) { - - /* If the caller provided a profile use that, otherwise make a new one */ - final DriverExecutionProfile profile = - profileOption.orElseGet( - () -> { - DriverExecutionProfile blankProfile = mock(DriverExecutionProfile.class); - when(blankProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION, "none")) - .thenReturn("none"); - when(blankProfile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) - .thenReturn(Duration.ofMinutes(5)); - when(blankProfile.isDefined(DefaultDriverOption.METRICS_FACTORY_CLASS)) - .thenReturn(true); - when(blankProfile.getString(DefaultDriverOption.METRICS_FACTORY_CLASS)) - .thenReturn("DefaultMetricsFactory"); - return blankProfile; - }); + DriverExecutionProfile defaultProfile, DriverExecutionProfile... profiles) { /* Setup machinery to connect the input DriverExecutionProfile to the config loader */ final DriverConfig driverConfig = mock(DriverConfig.class); final DriverConfigLoader configLoader = mock(DriverConfigLoader.class); when(configLoader.getInitialConfig()).thenReturn(driverConfig); - when(driverConfig.getDefaultProfile()).thenReturn(profile); + when(driverConfig.getDefaultProfile()).thenReturn(defaultProfile); + when(driverConfig.getProfile(defaultProfile.getName())).thenReturn(defaultProfile); + + for (DriverExecutionProfile profile : profiles) { + when(driverConfig.getProfile(profile.getName())).thenReturn(profile); + } ProgrammaticArguments args = ProgrammaticArguments.builder() @@ -71,6 +72,89 @@ public static DefaultDriverContext defaultDriverContext( .withLocalDatacenters(Maps.newHashMap()) .withNodeDistanceEvaluators(Maps.newHashMap()) .build(); - return new DefaultDriverContext(configLoader, args); + + return new DefaultDriverContext(configLoader, args) { + @NonNull + @Override + public Map getLoadBalancingPolicies() { + ImmutableMap.Builder map = ImmutableMap.builder(); + map.put( + defaultProfile.getName(), + mockLoadBalancingPolicy( + this, + defaultProfile.getName(), + defaultProfile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER))); + for (DriverExecutionProfile profile : profiles) { + map.put( + profile.getName(), + mockLoadBalancingPolicy( + this, + profile.getName(), + profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER))); + } + return map.build(); + } + + @NonNull + @Override + public ConsistencyLevelRegistry getConsistencyLevelRegistry() { + return mock(ConsistencyLevelRegistry.class); + } + }; + } + + public static DriverExecutionProfile defaultProfile(String localDc) { + return createProfile(DriverExecutionProfile.DEFAULT_NAME, localDc); + } + + public static DriverExecutionProfile createProfile(String name, String localDc) { + DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); + when(defaultProfile.getName()).thenReturn(name); + when(defaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION, "none")) + .thenReturn("none"); + when(defaultProfile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(Duration.ofMinutes(5)); + when(defaultProfile.isDefined(DefaultDriverOption.METRICS_FACTORY_CLASS)).thenReturn(true); + when(defaultProfile.getString(DefaultDriverOption.METRICS_FACTORY_CLASS)) + .thenReturn("DefaultMetricsFactory"); + when(defaultProfile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) + .thenReturn(localDc); + return defaultProfile; + } + + public static void allowRemoteDcConnectivity( + DriverExecutionProfile profile, + int maxNodesPerRemoteDc, + boolean allowRemoteSatisfyLocalDc, + List preferredRemoteDcs) { + when(profile.getInt(DefaultDriverOption.LOAD_BALANCING_DC_FAILOVER_MAX_NODES_PER_REMOTE_DC)) + .thenReturn(maxNodesPerRemoteDc); + when(profile.getBoolean( + DefaultDriverOption.LOAD_BALANCING_DC_FAILOVER_ALLOW_FOR_LOCAL_CONSISTENCY_LEVELS)) + .thenReturn(allowRemoteSatisfyLocalDc); + when(profile.getStringList(DefaultDriverOption.LOAD_BALANCING_DC_FAILOVER_PREFERRED_REMOTE_DCS)) + .thenReturn(preferredRemoteDcs); + } + + private static LoadBalancingPolicy mockLoadBalancingPolicy( + DefaultDriverContext driverContext, String profile, String localDc) { + LoadBalancingPolicy loadBalancingPolicy = + new DefaultLoadBalancingPolicy(driverContext, profile) { + @NonNull + @Override + protected Optional discoverLocalDc(@NonNull Map nodes) { + return Optional.ofNullable(localDc); + } + + @NonNull + @Override + protected NodeDistanceEvaluator createNodeDistanceEvaluator( + @Nullable String localDc, @NonNull Map nodes) { + return mock(NodeDistanceEvaluator.class); + } + }; + loadBalancingPolicy.init( + Collections.emptyMap(), mock(LoadBalancingPolicy.DistanceReporter.class)); + return loadBalancingPolicy; } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilderTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilderTest.java index 33811b2793a..d12e50b7e8e 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilderTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/context/StartupOptionsBuilderTest.java @@ -26,10 +26,10 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.session.Session; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.request.Startup; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,7 +41,8 @@ private DefaultDriverContext buildMockedContext(String compression) { DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); when(defaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION, "none")) .thenReturn(compression); - return MockedDriverContextFactory.defaultDriverContext(Optional.of(defaultProfile)); + when(defaultProfile.getName()).thenReturn(DriverExecutionProfile.DEFAULT_NAME); + return MockedDriverContextFactory.defaultDriverContext(defaultProfile); } private void assertDefaultStartupOptions(Startup startup) { @@ -94,4 +95,44 @@ public void should_fail_to_build_startup_options_with_invalid_compression() { new Startup(ctx.getStartupOptions()); }); } + + @Test + public void should_include_all_local_dcs_in_startup_message() { + + DefaultDriverContext ctx = + MockedDriverContextFactory.defaultDriverContext( + MockedDriverContextFactory.defaultProfile("us-west-2"), + MockedDriverContextFactory.createProfile("oltp", "us-east-2"), + MockedDriverContextFactory.createProfile("olap", "eu-central-1")); + Startup startup = new Startup(ctx.getStartupOptions()); + assertThat(startup.options) + .containsEntry( + StartupOptionsBuilder.DRIVER_BAGGAGE, + "{\"default\":{\"DefaultLoadBalancingPolicy\":{\"localDc\":\"us-west-2\"}}," + + "\"oltp\":{\"DefaultLoadBalancingPolicy\":{\"localDc\":\"us-east-2\"}}," + + "\"olap\":{\"DefaultLoadBalancingPolicy\":{\"localDc\":\"eu-central-1\"}}}"); + } + + @Test + public void should_include_all_lbp_details_in_startup_message() { + + DriverExecutionProfile defaultProfile = MockedDriverContextFactory.defaultProfile("dc1"); + DriverExecutionProfile oltpProfile = MockedDriverContextFactory.createProfile("oltp", "dc1"); + MockedDriverContextFactory.allowRemoteDcConnectivity( + oltpProfile, 2, true, ImmutableList.of("dc2", "dc3")); + DefaultDriverContext ctx = + MockedDriverContextFactory.defaultDriverContext(defaultProfile, oltpProfile); + + Startup startup = new Startup(ctx.getStartupOptions()); + + assertThat(startup.options) + .containsEntry( + StartupOptionsBuilder.DRIVER_BAGGAGE, + "{\"default\":{\"DefaultLoadBalancingPolicy\":{\"localDc\":\"dc1\"}}," + + "\"oltp\":{\"DefaultLoadBalancingPolicy\":{" + + "\"localDc\":\"dc1\"," + + "\"preferredRemoteDcs\":[\"dc2\",\"dc3\"]," + + "\"allowDcFailoverForLocalCl\":true," + + "\"maxNodesPerRemoteDc\":2}}}"); + } } From e2c7ad4d11555eeacce6bd436547b403f83eb24f Mon Sep 17 00:00:00 2001 From: janehe Date: Tue, 15 Apr 2025 13:18:34 -0700 Subject: [PATCH 66/77] CASSJAVA-97: Let users inject an ID for each request and write to the custom payload patch by Jane He; reviewed by Abe Ratnofsky and Bret McGuire for CASSJAVA-97 --- core/revapi.json | 5 + .../api/core/config/DefaultDriverOption.java | 6 + .../api/core/config/TypedDriverOption.java | 4 + .../api/core/context/DriverContext.java | 5 + .../core/session/ProgrammaticArguments.java | 17 +++ .../api/core/session/SessionBuilder.java | 22 +++ .../api/core/tracker/RequestIdGenerator.java | 77 +++++++++++ .../api/core/tracker/RequestTracker.java | 41 +++--- .../core/context/DefaultDriverContext.java | 24 ++++ .../internal/core/cql/CqlRequestHandler.java | 59 +++++++-- .../DefaultLoadBalancingPolicy.java | 4 +- .../tracker/MultiplexingRequestTracker.java | 31 +++-- .../core/tracker/NoopRequestTracker.java | 8 +- .../internal/core/tracker/RequestLogger.java | 12 +- .../core/tracker/UuidRequestIdGenerator.java | 43 ++++++ .../tracker/W3CContextRequestIdGenerator.java | 67 ++++++++++ core/src/main/resources/reference.conf | 7 + .../core/cql/RequestHandlerTestHarness.java | 3 + .../core/tracker/RequestIdGeneratorTest.java | 80 +++++++++++ .../core/tracker/RequestIdGeneratorIT.java | 125 ++++++++++++++++++ .../tracker/RequestNodeLoggerExample.java | 8 +- manual/core/request_id/README.md | 48 +++++++ 22 files changed, 637 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/tracker/UuidRequestIdGenerator.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/tracker/W3CContextRequestIdGenerator.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/tracker/RequestIdGeneratorTest.java create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java create mode 100644 manual/core/request_id/README.md diff --git a/core/revapi.json b/core/revapi.json index f39c7d4a7c0..8c707659c13 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -7407,6 +7407,11 @@ { "code": "java.method.varargOverloadsOnlyDifferInVarargParameter", "justification": "CASSJAVA-102: Migrate revapi config into dedicated config files, ported from pom.xml" + }, + { + "code": "java.method.addedToInterface", + "new": "method java.util.Optional com.datastax.oss.driver.api.core.context.DriverContext::getRequestIdGenerator()", + "justification": "CASSJAVA-97: Let users inject an ID for each request and write to the custom payload" } ] } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 4e45bf7b117..60c44193577 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -995,6 +995,12 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: boolean */ SSL_ALLOW_DNS_REVERSE_LOOKUP_SAN("advanced.ssl-engine-factory.allow-dns-reverse-lookup-san"), + /** + * The class of session-wide component that generates request IDs. + * + *

Value-type: {@link String} + */ + REQUEST_ID_GENERATOR_CLASS("advanced.request-id.generator.class"), /** * An address to always translate all node addresses to that same proxy hostname no matter what IP * address a node has, but still using its native transport port. diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index aa4e4af12dc..182753300e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -281,6 +281,10 @@ public String toString() { new TypedDriverOption<>( DefaultDriverOption.REQUEST_TRACKER_CLASSES, GenericType.listOf(String.class)); + /** The class of a session-wide component that generates request IDs. */ + public static final TypedDriverOption REQUEST_ID_GENERATOR_CLASS = + new TypedDriverOption<>(DefaultDriverOption.REQUEST_ID_GENERATOR_CLASS, GenericType.STRING); + /** Whether to log successful requests. */ public static final TypedDriverOption REQUEST_LOGGER_SUCCESS_ENABLED = new TypedDriverOption<>( diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java b/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java index 5b32389e362..6f0afd3df8a 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java @@ -33,6 +33,7 @@ import com.datastax.oss.driver.api.core.specex.SpeculativeExecutionPolicy; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; import com.datastax.oss.driver.api.core.time.TimestampGenerator; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.tracker.RequestTracker; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Map; @@ -139,6 +140,10 @@ default SpeculativeExecutionPolicy getSpeculativeExecutionPolicy(@NonNull String @NonNull RequestTracker getRequestTracker(); + /** @return The driver's request ID generator; never {@code null}. */ + @NonNull + Optional getRequestIdGenerator(); + /** @return The driver's request throttler; never {@code null}. */ @NonNull RequestThrottler getRequestThrottler(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java index 4e08bd5434c..5e10fb4d915 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java @@ -23,6 +23,7 @@ import com.datastax.oss.driver.api.core.metadata.NodeStateListener; import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.tracker.RequestTracker; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry; @@ -59,6 +60,7 @@ public static Builder builder() { private final NodeStateListener nodeStateListener; private final SchemaChangeListener schemaChangeListener; private final RequestTracker requestTracker; + private final RequestIdGenerator requestIdGenerator; private final Map localDatacenters; private final Map> nodeFilters; private final Map nodeDistanceEvaluators; @@ -77,6 +79,7 @@ private ProgrammaticArguments( @Nullable NodeStateListener nodeStateListener, @Nullable SchemaChangeListener schemaChangeListener, @Nullable RequestTracker requestTracker, + @Nullable RequestIdGenerator requestIdGenerator, @NonNull Map localDatacenters, @NonNull Map> nodeFilters, @NonNull Map nodeDistanceEvaluators, @@ -94,6 +97,7 @@ private ProgrammaticArguments( this.nodeStateListener = nodeStateListener; this.schemaChangeListener = schemaChangeListener; this.requestTracker = requestTracker; + this.requestIdGenerator = requestIdGenerator; this.localDatacenters = localDatacenters; this.nodeFilters = nodeFilters; this.nodeDistanceEvaluators = nodeDistanceEvaluators; @@ -128,6 +132,11 @@ public RequestTracker getRequestTracker() { return requestTracker; } + @Nullable + public RequestIdGenerator getRequestIdGenerator() { + return requestIdGenerator; + } + @NonNull public Map getLocalDatacenters() { return localDatacenters; @@ -196,6 +205,7 @@ public static class Builder { private NodeStateListener nodeStateListener; private SchemaChangeListener schemaChangeListener; private RequestTracker requestTracker; + private RequestIdGenerator requestIdGenerator; private ImmutableMap.Builder localDatacentersBuilder = ImmutableMap.builder(); private final ImmutableMap.Builder> nodeFiltersBuilder = ImmutableMap.builder(); @@ -294,6 +304,12 @@ public Builder addRequestTracker(@NonNull RequestTracker requestTracker) { return this; } + @NonNull + public Builder withRequestIdGenerator(@Nullable RequestIdGenerator requestIdGenerator) { + this.requestIdGenerator = requestIdGenerator; + return this; + } + @NonNull public Builder withLocalDatacenter( @NonNull String profileName, @NonNull String localDatacenter) { @@ -417,6 +433,7 @@ public ProgrammaticArguments build() { nodeStateListener, schemaChangeListener, requestTracker, + requestIdGenerator, localDatacentersBuilder.build(), nodeFiltersBuilder.build(), nodeDistanceEvaluatorsBuilder.build(), diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index cbf896a0873..25500119047 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -35,6 +35,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; import com.datastax.oss.driver.api.core.ssl.ProgrammaticSslEngineFactory; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.tracker.RequestTracker; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry; @@ -47,6 +48,7 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.session.DefaultSession; +import com.datastax.oss.driver.internal.core.tracker.W3CContextRequestIdGenerator; import com.datastax.oss.driver.internal.core.util.concurrent.BlockingOperation; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import edu.umd.cs.findbugs.annotations.NonNull; @@ -83,6 +85,8 @@ @NotThreadSafe public abstract class SessionBuilder { + public static final String ASTRA_PAYLOAD_KEY = "traceparent"; + private static final Logger LOG = LoggerFactory.getLogger(SessionBuilder.class); @SuppressWarnings("unchecked") @@ -318,6 +322,17 @@ public SelfT addRequestTracker(@NonNull RequestTracker requestTracker) { return self; } + /** + * Registers a request ID generator. The driver will use the generated ID in the logs and + * optionally add to the custom payload so that users can correlate logs about the same request + * from the Cassandra side. + */ + @NonNull + public SelfT withRequestIdGenerator(@NonNull RequestIdGenerator requestIdGenerator) { + this.programmaticArgumentsBuilder.withRequestIdGenerator(requestIdGenerator); + return self; + } + /** * Registers an authentication provider to use with the session. * @@ -861,6 +876,13 @@ protected final CompletionStage buildDefaultSessionAsync() { List configContactPoints = defaultConfig.getStringList(DefaultDriverOption.CONTACT_POINTS, Collections.emptyList()); if (cloudConfigInputStream != null) { + // override request id generator, unless user has already set it + if (programmaticArguments.getRequestIdGenerator() == null) { + programmaticArgumentsBuilder.withRequestIdGenerator( + new W3CContextRequestIdGenerator(ASTRA_PAYLOAD_KEY)); + LOG.debug( + "A secure connect bundle is provided, using W3CContextRequestIdGenerator as request ID generator."); + } if (!programmaticContactPoints.isEmpty() || !configContactPoints.isEmpty()) { LOG.info( "Both a secure connect bundle and contact points were provided. These are mutually exclusive. The contact points from the secure bundle will have priority."); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java new file mode 100644 index 00000000000..59ac3fdacf7 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.tracker; + +import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * Interface responsible for generating request IDs. + * + *

Note that all request IDs have a parent/child relationship. A "parent ID" can loosely be + * thought of as encompassing a sequence of a request + any attendant retries, speculative + * executions etc. It's scope is identical to that of a {@link + * com.datastax.oss.driver.internal.core.cql.CqlRequestHandler}. A "request ID" represents a single + * request within this larger scope. Note that a request corresponding to a request ID may be + * retried; in that case the retry count will be appended to the corresponding identifier in the + * logs. + */ +public interface RequestIdGenerator { + + String DEFAULT_PAYLOAD_KEY = "request-id"; + + /** + * Generates a unique identifier for the session request. This will be the identifier for the + * entire `session.execute()` call. This identifier will be added to logs, and propagated to + * request trackers. + * + * @return a unique identifier for the session request + */ + String getSessionRequestId(); + + /** + * Generates a unique identifier for the node request. This will be the identifier for the CQL + * request against a particular node. There can be one or more node requests for a single session + * request, due to retries or speculative executions. This identifier will be added to logs, and + * propagated to request trackers. + * + * @param statement the statement to be executed + * @param parentId the session request identifier + * @return a unique identifier for the node request + */ + String getNodeRequestId(@NonNull Request statement, @NonNull String parentId); + + default String getCustomPayloadKey() { + return DEFAULT_PAYLOAD_KEY; + } + + default Statement getDecoratedStatement( + @NonNull Statement statement, @NonNull String requestId) { + Map customPayload = + NullAllowingImmutableMap.builder() + .putAll(statement.getCustomPayload()) + .put(getCustomPayloadKey(), ByteBuffer.wrap(requestId.getBytes(StandardCharsets.UTF_8))) + .build(); + return statement.setCustomPayload(customPayload); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestTracker.java b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestTracker.java index d29ee48d352..065b41e496a 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestTracker.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestTracker.java @@ -47,21 +47,22 @@ default void onSuccess( @NonNull Node node) {} /** - * Invoked each time a request succeeds. + * Invoked each time a session request succeeds. A session request is a `session.execute()` call * * @param latencyNanos the overall execution time (from the {@link Session#execute(Request, * GenericType) session.execute} call until the result is made available to the client). * @param executionProfile the execution profile of this request. * @param node the node that returned the successful response. - * @param requestLogPrefix the dedicated log prefix for this request + * @param sessionRequestLogPrefix the dedicated log prefix for this request */ default void onSuccess( @NonNull Request request, long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestLogPrefix) { - // If client doesn't override onSuccess with requestLogPrefix delegate call to the old method + @NonNull String sessionRequestLogPrefix) { + // If client doesn't override onSuccess with sessionRequestLogPrefix delegate call to the old + // method onSuccess(request, latencyNanos, executionProfile, node); } @@ -78,13 +79,13 @@ default void onError( @Nullable Node node) {} /** - * Invoked each time a request fails. + * Invoked each time a session request fails. A session request is a `session.execute()` call * * @param latencyNanos the overall execution time (from the {@link Session#execute(Request, * GenericType) session.execute} call until the error is propagated to the client). * @param executionProfile the execution profile of this request. * @param node the node that returned the error response, or {@code null} if the error occurred - * @param requestLogPrefix the dedicated log prefix for this request + * @param sessionRequestLogPrefix the dedicated log prefix for this request */ default void onError( @NonNull Request request, @@ -92,8 +93,9 @@ default void onError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @Nullable Node node, - @NonNull String requestLogPrefix) { - // If client doesn't override onError with requestLogPrefix delegate call to the old method + @NonNull String sessionRequestLogPrefix) { + // If client doesn't override onError with sessionRequestLogPrefix delegate call to the old + // method onError(request, error, latencyNanos, executionProfile, node); } @@ -110,14 +112,15 @@ default void onNodeError( @NonNull Node node) {} /** - * Invoked each time a request fails at the node level. Similar to {@link #onError(Request, - * Throwable, long, DriverExecutionProfile, Node, String)} but at a per node level. + * Invoked each time a node request fails. A node request is a CQL request sent to a particular + * node. There can be one or more node requests for a single session request, due to retries or + * speculative executions. * * @param latencyNanos the overall execution time (from the {@link Session#execute(Request, * GenericType) session.execute} call until the error is propagated to the client). * @param executionProfile the execution profile of this request. * @param node the node that returned the error response. - * @param requestLogPrefix the dedicated log prefix for this request + * @param nodeRequestLogPrefix the dedicated log prefix for this request */ default void onNodeError( @NonNull Request request, @@ -125,8 +128,9 @@ default void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestLogPrefix) { - // If client doesn't override onNodeError with requestLogPrefix delegate call to the old method + @NonNull String nodeRequestLogPrefix) { + // If client doesn't override onNodeError with nodeRequestLogPrefix delegate call to the old + // method onNodeError(request, error, latencyNanos, executionProfile, node); } @@ -142,22 +146,23 @@ default void onNodeSuccess( @NonNull Node node) {} /** - * Invoked each time a request succeeds at the node level. Similar to {@link #onSuccess(Request, - * long, DriverExecutionProfile, Node, String)} but at per node level. + * Invoked each time a node request succeeds. A node request is a CQL request sent to a particular + * node. There can be one or more node requests for a single session request, due to retries or + * speculative executions. * * @param latencyNanos the overall execution time (from the {@link Session#execute(Request, * GenericType) session.execute} call until the result is made available to the client). * @param executionProfile the execution profile of this request. * @param node the node that returned the successful response. - * @param requestLogPrefix the dedicated log prefix for this request + * @param nodeRequestLogPrefix the dedicated log prefix for this request */ default void onNodeSuccess( @NonNull Request request, long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestLogPrefix) { - // If client doesn't override onNodeSuccess with requestLogPrefix delegate call to the old + @NonNull String nodeRequestLogPrefix) { + // If client doesn't override onNodeSuccess with nodeRequestLogPrefix delegate call to the old // method onNodeSuccess(request, latencyNanos, executionProfile, node); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java index 0d7db27dfbe..3074bda2398 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java @@ -44,6 +44,7 @@ import com.datastax.oss.driver.api.core.specex.SpeculativeExecutionPolicy; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; import com.datastax.oss.driver.api.core.time.TimestampGenerator; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.tracker.RequestTracker; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; @@ -221,6 +222,7 @@ public class DefaultDriverContext implements InternalDriverContext { private final LazyReference nodeStateListenerRef; private final LazyReference schemaChangeListenerRef; private final LazyReference requestTrackerRef; + private final LazyReference> requestIdGeneratorRef; private final LazyReference> authProviderRef; private final LazyReference> lifecycleListenersRef = new LazyReference<>("lifecycleListeners", this::buildLifecycleListeners, cycleDetector); @@ -282,6 +284,11 @@ public DefaultDriverContext( this.requestTrackerRef = new LazyReference<>( "requestTracker", () -> buildRequestTracker(requestTrackerFromBuilder), cycleDetector); + this.requestIdGeneratorRef = + new LazyReference<>( + "requestIdGenerator", + () -> buildRequestIdGenerator(programmaticArguments.getRequestIdGenerator()), + cycleDetector); this.sslEngineFactoryRef = new LazyReference<>( "sslEngineFactory", @@ -708,6 +715,17 @@ protected RequestTracker buildRequestTracker(RequestTracker requestTrackerFromBu } } + protected Optional buildRequestIdGenerator( + RequestIdGenerator requestIdGenerator) { + return (requestIdGenerator != null) + ? Optional.of(requestIdGenerator) + : Reflection.buildFromConfig( + this, + DefaultDriverOption.REQUEST_ID_GENERATOR_CLASS, + RequestIdGenerator.class, + "com.datastax.oss.driver.internal.core.tracker"); + } + protected Optional buildAuthProvider(AuthProvider authProviderFromBuilder) { return (authProviderFromBuilder != null) ? Optional.of(authProviderFromBuilder) @@ -972,6 +990,12 @@ public RequestTracker getRequestTracker() { return requestTrackerRef.get(); } + @NonNull + @Override + public Optional getRequestIdGenerator() { + return requestIdGeneratorRef.get(); + } + @Nullable @Override public String getLocalDatacenter(@NonNull String profileName) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index 0808bdce63f..6842547b11a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -44,6 +44,7 @@ import com.datastax.oss.driver.api.core.servererrors.WriteTimeoutException; import com.datastax.oss.driver.api.core.session.throttling.RequestThrottler; import com.datastax.oss.driver.api.core.session.throttling.Throttled; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.tracker.RequestTracker; import com.datastax.oss.driver.internal.core.adminrequest.ThrottledAdminRequestHandler; import com.datastax.oss.driver.internal.core.adminrequest.UnexpectedResponseException; @@ -59,6 +60,7 @@ import com.datastax.oss.driver.internal.core.tracker.RequestLogger; import com.datastax.oss.driver.internal.core.util.Loggers; import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; +import com.datastax.oss.driver.shaded.guava.common.base.Joiner; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -82,6 +84,7 @@ import java.util.AbstractMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -100,7 +103,7 @@ public class CqlRequestHandler implements Throttled { private static final long NANOTIME_NOT_MEASURED_YET = -1; private final long startTimeNanos; - private final String logPrefix; + private final String handlerLogPrefix; private final Statement initialStatement; private final DefaultSession session; private final CqlIdentifier keyspace; @@ -125,6 +128,7 @@ public class CqlRequestHandler implements Throttled { private final List inFlightCallbacks; private final RequestThrottler throttler; private final RequestTracker requestTracker; + private final Optional requestIdGenerator; private final SessionMetricUpdater sessionMetricUpdater; private final DriverExecutionProfile executionProfile; @@ -132,15 +136,25 @@ public class CqlRequestHandler implements Throttled { // We don't use a map because nodes can appear multiple times. private volatile List> errors; + private final Joiner logPrefixJoiner = Joiner.on('|'); + private final String sessionName; + private final String sessionRequestId; + protected CqlRequestHandler( Statement statement, DefaultSession session, InternalDriverContext context, - String sessionLogPrefix) { + String sessionName) { this.startTimeNanos = System.nanoTime(); - this.logPrefix = sessionLogPrefix + "|" + this.hashCode(); - LOG.trace("[{}] Creating new handler for request {}", logPrefix, statement); + this.requestIdGenerator = context.getRequestIdGenerator(); + this.sessionName = sessionName; + this.sessionRequestId = + this.requestIdGenerator + .map(RequestIdGenerator::getSessionRequestId) + .orElse(Integer.toString(this.hashCode())); + this.handlerLogPrefix = logPrefixJoiner.join(sessionName, sessionRequestId); + LOG.trace("[{}] Creating new handler for request {}", handlerLogPrefix, statement); this.initialStatement = statement; this.session = session; @@ -155,7 +169,7 @@ protected CqlRequestHandler( context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { - Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); + Loggers.warnWithException(LOG, "[{}] Uncaught exception", handlerLogPrefix, t2); } return null; }); @@ -250,9 +264,9 @@ private void sendRequest( } Node node = retriedNode; DriverChannel channel = null; - if (node == null || (channel = session.getChannel(node, logPrefix)) == null) { + if (node == null || (channel = session.getChannel(node, handlerLogPrefix)) == null) { while (!result.isDone() && (node = queryPlan.poll()) != null) { - channel = session.getChannel(node, logPrefix); + channel = session.getChannel(node, handlerLogPrefix); if (channel != null) { break; } else { @@ -267,6 +281,16 @@ private void sendRequest( setFinalError(statement, AllNodesFailedException.fromErrors(this.errors), null, -1); } } else { + Statement finalStatement = statement; + String nodeRequestId = + this.requestIdGenerator + .map((g) -> g.getNodeRequestId(finalStatement, sessionRequestId)) + .orElse(Integer.toString(this.hashCode())); + statement = + this.requestIdGenerator + .map((g) -> g.getDecoratedStatement(finalStatement, nodeRequestId)) + .orElse(finalStatement); + NodeResponseCallback nodeResponseCallback = new NodeResponseCallback( statement, @@ -276,7 +300,7 @@ private void sendRequest( currentExecutionIndex, retryCount, scheduleNextExecution, - logPrefix); + logPrefixJoiner.join(this.sessionName, nodeRequestId, currentExecutionIndex)); Message message = Conversions.toMessage(statement, executionProfile, context); channel .write(message, statement.isTracing(), statement.getCustomPayload(), nodeResponseCallback) @@ -335,9 +359,17 @@ private void setFinalResult( totalLatencyNanos = completionTimeNanos - startTimeNanos; long nodeLatencyNanos = completionTimeNanos - callback.nodeStartTimeNanos; requestTracker.onNodeSuccess( - callback.statement, nodeLatencyNanos, executionProfile, callback.node, logPrefix); + callback.statement, + nodeLatencyNanos, + executionProfile, + callback.node, + handlerLogPrefix); requestTracker.onSuccess( - callback.statement, totalLatencyNanos, executionProfile, callback.node, logPrefix); + callback.statement, + totalLatencyNanos, + executionProfile, + callback.node, + handlerLogPrefix); } if (sessionMetricUpdater.isEnabled( DefaultSessionMetric.CQL_REQUESTS, executionProfile.getName())) { @@ -439,7 +471,8 @@ private void setFinalError(Statement statement, Throwable error, Node node, i cancelScheduledTasks(); if (!(requestTracker instanceof NoopRequestTracker)) { long latencyNanos = System.nanoTime() - startTimeNanos; - requestTracker.onError(statement, error, latencyNanos, executionProfile, node, logPrefix); + requestTracker.onError( + statement, error, latencyNanos, executionProfile, node, handlerLogPrefix); } if (error instanceof DriverTimeoutException) { throttler.signalTimeout(this); @@ -489,7 +522,7 @@ private NodeResponseCallback( this.execution = execution; this.retryCount = retryCount; this.scheduleNextExecution = scheduleNextExecution; - this.logPrefix = logPrefix + "|" + execution; + this.logPrefix = logPrefix; } // this gets invoked once the write completes. @@ -567,7 +600,7 @@ private void scheduleSpeculativeExecution(int index, long delay) { if (!result.isDone()) { LOG.trace( "[{}] Starting speculative execution {}", - CqlRequestHandler.this.logPrefix, + CqlRequestHandler.this.handlerLogPrefix, index); activeExecutionsCount.incrementAndGet(); startedSpeculativeExecutionsCount.incrementAndGet(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java index 9c31b606f18..8e1c1fe5039 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java @@ -246,7 +246,7 @@ public void onNodeSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { updateResponseTimes(node); } @@ -257,7 +257,7 @@ public void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { updateResponseTimes(node); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/MultiplexingRequestTracker.java b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/MultiplexingRequestTracker.java index d4d20f3eb78..6fe2ba059bd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/MultiplexingRequestTracker.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/MultiplexingRequestTracker.java @@ -82,10 +82,12 @@ public void onSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String sessionRequestLogPrefix) { invokeTrackers( - tracker -> tracker.onSuccess(request, latencyNanos, executionProfile, node, logPrefix), - logPrefix, + tracker -> + tracker.onSuccess( + request, latencyNanos, executionProfile, node, sessionRequestLogPrefix), + sessionRequestLogPrefix, "onSuccess"); } @@ -96,10 +98,12 @@ public void onError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @Nullable Node node, - @NonNull String logPrefix) { + @NonNull String sessionRequestLogPrefix) { invokeTrackers( - tracker -> tracker.onError(request, error, latencyNanos, executionProfile, node, logPrefix), - logPrefix, + tracker -> + tracker.onError( + request, error, latencyNanos, executionProfile, node, sessionRequestLogPrefix), + sessionRequestLogPrefix, "onError"); } @@ -109,10 +113,12 @@ public void onNodeSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { invokeTrackers( - tracker -> tracker.onNodeSuccess(request, latencyNanos, executionProfile, node, logPrefix), - logPrefix, + tracker -> + tracker.onNodeSuccess( + request, latencyNanos, executionProfile, node, nodeRequestLogPrefix), + nodeRequestLogPrefix, "onNodeSuccess"); } @@ -123,11 +129,12 @@ public void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { invokeTrackers( tracker -> - tracker.onNodeError(request, error, latencyNanos, executionProfile, node, logPrefix), - logPrefix, + tracker.onNodeError( + request, error, latencyNanos, executionProfile, node, nodeRequestLogPrefix), + nodeRequestLogPrefix, "onNodeError"); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/NoopRequestTracker.java b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/NoopRequestTracker.java index 09ac27e5e75..3821c6ace2d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/NoopRequestTracker.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/NoopRequestTracker.java @@ -42,7 +42,7 @@ public void onSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestPrefix) { + @NonNull String sessionRequestLogPrefix) { // nothing to do } @@ -53,7 +53,7 @@ public void onError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, Node node, - @NonNull String requestPrefix) { + @NonNull String sessionRequestLogPrefix) { // nothing to do } @@ -64,7 +64,7 @@ public void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestPrefix) { + @NonNull String nodeRequestLogPrefix) { // nothing to do } @@ -74,7 +74,7 @@ public void onNodeSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String requestPrefix) { + @NonNull String nodeRequestLogPrefix) { // nothing to do } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/RequestLogger.java b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/RequestLogger.java index 235ef051b40..f242ff89c54 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/RequestLogger.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/RequestLogger.java @@ -86,7 +86,7 @@ public void onSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String sessionRequestLogPrefix) { boolean successEnabled = executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOGGER_SUCCESS_ENABLED, false); @@ -129,7 +129,7 @@ public void onSuccess( showValues, maxValues, maxValueLength, - logPrefix); + sessionRequestLogPrefix); } @Override @@ -139,7 +139,7 @@ public void onError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, Node node, - @NonNull String logPrefix) { + @NonNull String sessionRequestLogPrefix) { if (!executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOGGER_ERROR_ENABLED, false)) { return; @@ -173,7 +173,7 @@ public void onError( maxValues, maxValueLength, showStackTraces, - logPrefix); + sessionRequestLogPrefix); } @Override @@ -183,7 +183,7 @@ public void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { // Nothing to do } @@ -193,7 +193,7 @@ public void onNodeSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { // Nothing to do } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/UuidRequestIdGenerator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/UuidRequestIdGenerator.java new file mode 100644 index 00000000000..cc07d6717f4 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/UuidRequestIdGenerator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.tracker; + +import com.datastax.oss.driver.api.core.context.DriverContext; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; +import com.datastax.oss.driver.api.core.uuid.Uuids; +import edu.umd.cs.findbugs.annotations.NonNull; + +public class UuidRequestIdGenerator implements RequestIdGenerator { + public UuidRequestIdGenerator(DriverContext context) {} + + /** Generates a random v4 UUID. */ + @Override + public String getSessionRequestId() { + return Uuids.random().toString(); + } + + /** + * {session-request-id}-{random-uuid} All node requests for a session request will have the same + * session request id + */ + @Override + public String getNodeRequestId(@NonNull Request statement, @NonNull String parentId) { + return parentId + "-" + Uuids.random(); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/W3CContextRequestIdGenerator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/W3CContextRequestIdGenerator.java new file mode 100644 index 00000000000..fe15b93bc8e --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/tracker/W3CContextRequestIdGenerator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.tracker; + +import com.datastax.oss.driver.api.core.context.DriverContext; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; +import com.datastax.oss.driver.shaded.guava.common.io.BaseEncoding; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.security.SecureRandom; +import java.util.Random; + +public class W3CContextRequestIdGenerator implements RequestIdGenerator { + + private final Random random = new SecureRandom(); + private final BaseEncoding baseEncoding = BaseEncoding.base16().lowerCase(); + private final String payloadKey; + + public W3CContextRequestIdGenerator(DriverContext context) { + payloadKey = RequestIdGenerator.super.getCustomPayloadKey(); + } + + public W3CContextRequestIdGenerator(String payloadKey) { + this.payloadKey = payloadKey; + } + + /** Random 16 bytes, e.g. "4bf92f3577b34da6a3ce929d0e0e4736" */ + @Override + public String getSessionRequestId() { + byte[] bytes = new byte[16]; + random.nextBytes(bytes); + return baseEncoding.encode(bytes); + } + + /** + * Following the format of W3C "traceparent" spec, + * https://www.w3.org/TR/trace-context/#traceparent-header-field-values e.g. + * "00-4bf92f3577b34da6a3ce929d0e0e4736-a3ce929d0e0e4736-01" All node requests in the same session + * request share the same "trace-id" field value + */ + @Override + public String getNodeRequestId(@NonNull Request statement, @NonNull String parentId) { + byte[] bytes = new byte[8]; + random.nextBytes(bytes); + return String.format("00-%s-%s-00", parentId, baseEncoding.encode(bytes)); + } + + @Override + public String getCustomPayloadKey() { + return this.payloadKey; + } +} diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 3c6851a48ee..741b1d97654 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -918,6 +918,13 @@ datastax-java-driver { } } + advanced.request-id { + generator { + # The component that generates a unique identifier for each CQL request, and possibly write the id to the custom payload . + // class = W3CContextRequestIdGenerator + } + } + # A session-wide component that controls the rate at which requests are executed. # # Implementations vary, but throttlers generally track a metric that represents the level of diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java index 9d86302aabf..6ecd6111992 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java @@ -61,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -168,6 +169,8 @@ protected RequestHandlerTestHarness(Builder builder) { when(context.getRequestThrottler()).thenReturn(new PassThroughRequestThrottler(context)); when(context.getRequestTracker()).thenReturn(new NoopRequestTracker(context)); + + when(context.getRequestIdGenerator()).thenReturn(Optional.empty()); } public DefaultSession getSession() { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/tracker/RequestIdGeneratorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/tracker/RequestIdGeneratorTest.java new file mode 100644 index 00000000000..fb1883e125f --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/tracker/RequestIdGeneratorTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.tracker; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.Strict.class) +public class RequestIdGeneratorTest { + @Mock private InternalDriverContext context; + @Mock private Statement statement; + + @Test + public void uuid_generator_should_generate() { + // given + UuidRequestIdGenerator generator = new UuidRequestIdGenerator(context); + // when + String parentId = generator.getSessionRequestId(); + String requestId = generator.getNodeRequestId(statement, parentId); + // then + // e.g. "550e8400-e29b-41d4-a716-446655440000", which is 36 characters long + assertThat(parentId.length()).isEqualTo(36); + // e.g. "550e8400-e29b-41d4-a716-446655440000-550e8400-e29b-41d4-a716-446655440000", which is 73 + // characters long + assertThat(requestId.length()).isEqualTo(73); + } + + @Test + public void w3c_generator_should_generate() { + // given + W3CContextRequestIdGenerator generator = new W3CContextRequestIdGenerator(context); + // when + String parentId = generator.getSessionRequestId(); + String requestId = generator.getNodeRequestId(statement, parentId); + // then + // e.g. "4bf92f3577b34da6a3ce929d0e0e4736", which is 32 characters long + assertThat(parentId.length()).isEqualTo(32); + // According to W3C "traceparent" spec, + // https://www.w3.org/TR/trace-context/#traceparent-header-field-values + // e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-a3ce929d0e0e4736-01", which 55 characters long + assertThat(requestId.length()).isEqualTo(55); + } + + @Test + public void w3c_generator_default_payloadkey() { + W3CContextRequestIdGenerator w3cGenerator = new W3CContextRequestIdGenerator(context); + assertThat(w3cGenerator.getCustomPayloadKey()) + .isEqualTo(RequestIdGenerator.DEFAULT_PAYLOAD_KEY); + } + + @Test + public void w3c_generator_provided_payloadkey() { + String someString = RandomStringUtils.random(12); + W3CContextRequestIdGenerator w3cGenerator = new W3CContextRequestIdGenerator(someString); + assertThat(w3cGenerator.getCustomPayloadKey()).isEqualTo(someString); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java new file mode 100644 index 00000000000..2848a8fb629 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.core.tracker; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; +import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.ParallelizableTests; +import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@Category(ParallelizableTests.class) +public class RequestIdGeneratorIT { + private CcmRule ccmRule = CcmRule.getInstance(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule); + + @Test + public void should_write_uuid_to_custom_payload_with_key() { + DriverConfigLoader loader = + SessionUtils.configLoaderBuilder() + .withString(DefaultDriverOption.REQUEST_ID_GENERATOR_CLASS, "UuidRequestIdGenerator") + .build(); + try (CqlSession session = SessionUtils.newSession(ccmRule, loader)) { + String query = "SELECT * FROM system.local"; + ResultSet rs = session.execute(query); + ByteBuffer id = rs.getExecutionInfo().getRequest().getCustomPayload().get("request-id"); + assertThat(id.remaining()).isEqualTo(73); + } + } + + @Test + public void should_write_default_request_id_to_custom_payload_with_key() { + DriverConfigLoader loader = + SessionUtils.configLoaderBuilder() + .withString( + DefaultDriverOption.REQUEST_ID_GENERATOR_CLASS, "W3CContextRequestIdGenerator") + .build(); + try (CqlSession session = SessionUtils.newSession(ccmRule, loader)) { + String query = "SELECT * FROM system.local"; + ResultSet rs = session.execute(query); + ByteBuffer id = rs.getExecutionInfo().getRequest().getCustomPayload().get("request-id"); + assertThat(id.remaining()).isEqualTo(55); + } + } + + @Test + public void should_use_customized_request_id_generator() { + RequestIdGenerator myRequestIdGenerator = + new RequestIdGenerator() { + @Override + public String getSessionRequestId() { + return "123"; + } + + @Override + public String getNodeRequestId(@NonNull Request statement, @NonNull String parentId) { + return "456"; + } + + @Override + public Statement getDecoratedStatement( + @NonNull Statement statement, @NonNull String requestId) { + Map customPayload = + NullAllowingImmutableMap.builder() + .putAll(statement.getCustomPayload()) + .put("trace_key", ByteBuffer.wrap(requestId.getBytes(StandardCharsets.UTF_8))) + .build(); + return statement.setCustomPayload(customPayload); + } + }; + try (CqlSession session = + (CqlSession) + SessionUtils.baseBuilder() + .addContactEndPoints(ccmRule.getContactPoints()) + .withRequestIdGenerator(myRequestIdGenerator) + .build()) { + String query = "SELECT * FROM system.local"; + ResultSet rs = session.execute(query); + ByteBuffer id = rs.getExecutionInfo().getRequest().getCustomPayload().get("trace_key"); + assertThat(id).isEqualTo(ByteBuffer.wrap("456".getBytes(StandardCharsets.UTF_8))); + } + } + + @Test + public void should_not_write_id_to_custom_payload_when_key_is_not_set() { + DriverConfigLoader loader = SessionUtils.configLoaderBuilder().build(); + try (CqlSession session = SessionUtils.newSession(ccmRule, loader)) { + String query = "SELECT * FROM system.local"; + ResultSet rs = session.execute(query); + assertThat(rs.getExecutionInfo().getRequest().getCustomPayload().get("trace_key")).isNull(); + } + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestNodeLoggerExample.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestNodeLoggerExample.java index eae98339637..8eb2fb80a73 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestNodeLoggerExample.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestNodeLoggerExample.java @@ -39,7 +39,7 @@ public void onNodeError( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { if (!executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOGGER_ERROR_ENABLED)) { return; } @@ -66,7 +66,7 @@ public void onNodeError( maxValues, maxValueLength, showStackTraces, - logPrefix); + nodeRequestLogPrefix); } @Override @@ -75,7 +75,7 @@ public void onNodeSuccess( long latencyNanos, @NonNull DriverExecutionProfile executionProfile, @NonNull Node node, - @NonNull String logPrefix) { + @NonNull String nodeRequestLogPrefix) { boolean successEnabled = executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOGGER_SUCCESS_ENABLED); boolean slowEnabled = @@ -114,6 +114,6 @@ public void onNodeSuccess( showValues, maxValues, maxValueLength, - logPrefix); + nodeRequestLogPrefix); } } diff --git a/manual/core/request_id/README.md b/manual/core/request_id/README.md new file mode 100644 index 00000000000..a766a4419af --- /dev/null +++ b/manual/core/request_id/README.md @@ -0,0 +1,48 @@ + + +## Request Id + +### Quick overview + +Users can inject an identifier for each individual CQL request, and such ID can be written in to the [custom payload](https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v5.spec) to +correlate a request across the driver and the Apache Cassandra server. + +A request ID generator needs to generate both: +- Session request ID: an identifier for an entire session.execute() call +- Node request ID: an identifier for the execution of a CQL statement against a particular node. There can be one or more node requests for a single session request, due to retries or speculative executions. + +Usage: +* Inject ID generator: set the desired `RequestIdGenerator` in `advanced.request-id.generator.class`. +* Add ID to custom payload: the default behavior of a `RequestIdGenerator` is to add the request ID into the custom payload with the key `request-id`. Override `RequestIdGenerator.getDecoratedStatement` to customize the behavior. + +### Request Id Generator Configuration + +Request ID generator can be declared in the [configuration](../configuration/) as follows: + +``` +datastax-java-driver.advanced.request-id.generator { + class = com.example.app.MyGenerator +} +``` + +To register your own request ID generator, specify the name of the class +that implements `RequestIdGenerator`. + +The generated ID will be added to the log message of `CqlRequestHandler`, and propagated to other classes, e.g. the request trackers. \ No newline at end of file From 104751c166402274f46135b9d367bee1cfdd124d Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 14 Oct 2025 15:08:52 -0500 Subject: [PATCH 67/77] Changelog entries for 4.19.1 patch by Bret McGuire; reviewed by Andy Tolbert --- changelog/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/changelog/README.md b/changelog/README.md index 08634bcb834..9e223318e65 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -21,6 +21,24 @@ under the License. +### 4.19.1 + +- [improvement] CASSJAVA-97: Let users inject an ID for each request and write to the custom payload +- [improvement] CASSJAVA-92: Add Local DC to driver connection info and provide visibility with nodetool clientstats +- [bug] PR 2025: Eliminate lock in ConcurrencyLimitingRequestThrottler +- [improvement] CASSJAVA-89: Fix deprecated table configs in Cassandra 5 +- [improvement] PR 2028: Remove unnecessary locking in DefaultNettyOptions +- [improvement] CASSJAVA-102: Fix revapi spurious complaints about optional dependencies +- [improvement] PR 2013: Add SubnetAddressTranslator +- [improvement] CASSJAVA-68: Improve DefaultCodecRegistry.CacheKey#hashCode() to eliminate Object[] allocation +- [improvement] PR 1989: Bump Jackson version to la(te)st 2.13.x, 2.13.5 +- [improvement] CASSJAVA-76: Make guava an optional dependency of java-driver-guava-shaded +- [bug] PR 2035: Prevent long overflow in SNI address resolution +- [improvement] CASSJAVA-77: 4.x: Upgrade Netty to 4.1.119 +- [improvement] CASSJAVA-40: Driver testing against Java 21 +- [improvement] CASSJAVA-90: Update native-protocol +- [improvement] CASSJAVA-80: Support configuration to disable DNS reverse-lookups for SAN validation + ### 4.19.0 - [bug] JAVA-3055: Prevent PreparedStatement cache to be polluted if a request is cancelled. From 77b2baebebfa208e2a7a29f6b09b6cb86e3a4b61 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Tue, 14 Oct 2025 15:32:38 -0500 Subject: [PATCH 68/77] [maven-release-plugin] prepare release 4.19.1 --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index f03317edc03..05e9d74dc5c 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-core-shaded - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-mapper-processor - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-mapper-runtime - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-query-builder - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-guava-shaded - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-test-infra - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-metrics-micrometer - 4.19.1-SNAPSHOT + 4.19.1 org.apache.cassandra java-driver-metrics-microprofile - 4.19.1-SNAPSHOT + 4.19.1 com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 451db1dcd1b..4cc8197b6dd 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index b8d7d5c2d3b..9ed02b81dd7 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index e930f4c0610..5328b4bbd36 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index 1c762074673..c4b5eb38026 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index 8f7740e148f..4c5f04de515 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index 15f082e6864..a473320ea44 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.1-SNAPSHOT + 4.19.1 java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index ed37e861a96..62c1c5fc799 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index b489076c257..16661191427 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 519f1411ce9..d7aaa7f9d67 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index 3a767c2a352..af2936a3805 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 091bb5f3e93..3dd89d1737b 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 5163b5366f4..cca6eafc603 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 4b41f790145..e60981f7fc8 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index 2cfeb65e757..355449acf0e 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1055,7 +1055,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - HEAD + 4.19.1 diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 0bc46f9bb91..1860cf65d12 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index b0808757ce4..e61f4f2826b 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1-SNAPSHOT + 4.19.1 java-driver-test-infra bundle From f63108175fd22e2db893c57b0975bc518977ab4b Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Tue, 14 Oct 2025 15:32:41 -0500 Subject: [PATCH 69/77] [maven-release-plugin] prepare for next development iteration --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 05e9d74dc5c..fab36abad4c 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-core-shaded - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-mapper-processor - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-mapper-runtime - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-query-builder - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-guava-shaded - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-test-infra - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-metrics-micrometer - 4.19.1 + 4.19.2-SNAPSHOT org.apache.cassandra java-driver-metrics-microprofile - 4.19.1 + 4.19.2-SNAPSHOT com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 4cc8197b6dd..617664eec97 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index 9ed02b81dd7..e750b791d3b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index 5328b4bbd36..27d6d026a4d 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index c4b5eb38026..fbf8cd21076 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index 4c5f04de515..498d5dc603a 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index a473320ea44..df1e2b2613b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.1 + 4.19.2-SNAPSHOT java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 62c1c5fc799..ad581bc0f98 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 16661191427..8360c8a211f 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index d7aaa7f9d67..5368c24c2b6 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index af2936a3805..a21254da908 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 3dd89d1737b..08211d04c2b 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index cca6eafc603..95c1ca9bb42 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index e60981f7fc8..abb54e159f6 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index 355449acf0e..4e3d7006169 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1055,7 +1055,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - 4.19.1 + HEAD diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 1860cf65d12..e1db8485799 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index e61f4f2826b..1a73cde23cd 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.1 + 4.19.2-SNAPSHOT java-driver-test-infra bundle From 8f69c24d6cd6db75c6811fc32ee9ab39121ecaa6 Mon Sep 17 00:00:00 2001 From: janehe Date: Mon, 10 Nov 2025 12:50:59 -0800 Subject: [PATCH 70/77] CASSJAVA-116: Retry or Speculative Execution with RequestIdGenerator throws "Duplicate Key" patch by Jane He; reviewed by Andy Tolbert and Lukasz Atoniak for CASSJAVA-116 --- .../api/core/tracker/RequestIdGenerator.java | 29 +++++---- .../core/cql/CqlRequestHandlerRetryTest.java | 63 +++++++++++++++++++ .../core/cql/RequestHandlerTestHarness.java | 10 ++- .../core/tracker/RequestIdGeneratorIT.java | 21 ++++++- 4 files changed, 110 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java index 59ac3fdacf7..21db3793b01 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/tracker/RequestIdGenerator.java @@ -19,20 +19,21 @@ import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.core.session.Request; -import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; import edu.umd.cs.findbugs.annotations.NonNull; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; /** * Interface responsible for generating request IDs. * - *

Note that all request IDs have a parent/child relationship. A "parent ID" can loosely be - * thought of as encompassing a sequence of a request + any attendant retries, speculative + *

Note that all request IDs have a parent/child relationship. A "session request ID" can loosely + * be thought of as encompassing a sequence of a request + any attendant retries, speculative * executions etc. It's scope is identical to that of a {@link - * com.datastax.oss.driver.internal.core.cql.CqlRequestHandler}. A "request ID" represents a single - * request within this larger scope. Note that a request corresponding to a request ID may be + * com.datastax.oss.driver.internal.core.cql.CqlRequestHandler}. A "node request ID" represents a + * single request within this larger scope. Note that a request corresponding to a request ID may be * retried; in that case the retry count will be appended to the corresponding identifier in the * logs. */ @@ -67,11 +68,17 @@ default String getCustomPayloadKey() { default Statement getDecoratedStatement( @NonNull Statement statement, @NonNull String requestId) { - Map customPayload = - NullAllowingImmutableMap.builder() - .putAll(statement.getCustomPayload()) - .put(getCustomPayloadKey(), ByteBuffer.wrap(requestId.getBytes(StandardCharsets.UTF_8))) - .build(); - return statement.setCustomPayload(customPayload); + + Map existing = new HashMap<>(statement.getCustomPayload()); + String key = getCustomPayloadKey(); + + // Add or overwrite + existing.put(key, ByteBuffer.wrap(requestId.getBytes(StandardCharsets.UTF_8))); + + // Allowing null key/values + // Wrap a map inside to be immutable without instanciating a new map + Map unmodifiableMap = Collections.unmodifiableMap(existing); + + return statement.setCustomPayload(unmodifiableMap); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandlerRetryTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandlerRetryTest.java index bea52891c18..ccac873c616 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandlerRetryTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandlerRetryTest.java @@ -48,6 +48,8 @@ import com.datastax.oss.driver.api.core.servererrors.ServerError; import com.datastax.oss.driver.api.core.servererrors.UnavailableException; import com.datastax.oss.driver.api.core.servererrors.WriteTimeoutException; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.response.Error; import com.datastax.oss.protocol.internal.response.error.ReadTimeout; @@ -55,9 +57,13 @@ import com.datastax.oss.protocol.internal.response.error.WriteTimeout; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.concurrent.CompletionStage; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; public class CqlRequestHandlerRetryTest extends CqlRequestHandlerTestBase { @@ -384,6 +390,63 @@ public void should_rethrow_error_if_not_idempotent_and_error_unsafe_or_policy_re } } + @Test + @UseDataProvider("failureAndIdempotent") + public void should_not_fail_with_duplicate_key_when_retrying_with_request_id_generator( + FailureScenario failureScenario, boolean defaultIdempotence, Statement statement) { + + // Create a RequestIdGenerator that uses the same key as the statement's custom payload + RequestIdGenerator requestIdGenerator = + new RequestIdGenerator() { + private AtomicInteger counter = new AtomicInteger(0); + + @Override + public String getSessionRequestId() { + return "session-123"; + } + + @Override + public String getNodeRequestId(@NonNull Request request, @NonNull String parentId) { + return parentId + "-" + counter.getAndIncrement(); + } + }; + + RequestHandlerTestHarness.Builder harnessBuilder = + RequestHandlerTestHarness.builder() + .withDefaultIdempotence(defaultIdempotence) + .withRequestIdGenerator(requestIdGenerator); + failureScenario.mockRequestError(harnessBuilder, node1); + harnessBuilder.withResponse(node2, defaultFrameOf(singleRow())); + + try (RequestHandlerTestHarness harness = harnessBuilder.build()) { + failureScenario.mockRetryPolicyVerdict( + harness.getContext().getRetryPolicy(anyString()), RetryVerdict.RETRY_NEXT); + + CompletionStage resultSetFuture = + new CqlRequestHandler(statement, harness.getSession(), harness.getContext(), "test") + .handle(); + + // The test should succeed without throwing a duplicate key exception + assertThatStage(resultSetFuture) + .isSuccess( + resultSet -> { + Iterator rows = resultSet.currentPage().iterator(); + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getString("message")).isEqualTo("hello, world"); + + ExecutionInfo executionInfo = resultSet.getExecutionInfo(); + assertThat(executionInfo.getCoordinator()).isEqualTo(node2); + assertThat(executionInfo.getErrors()).hasSize(1); + assertThat(executionInfo.getErrors().get(0).getKey()).isEqualTo(node1); + + // Verify that the custom payload still contains the request ID key + // (either the original value or the generated one, depending on implementation) + assertThat(executionInfo.getRequest().getCustomPayload().get("request-id")) + .isEqualTo(ByteBuffer.wrap("session-123-1".getBytes(StandardCharsets.UTF_8))); + }); + } + } + /** * Sets up the mocks to simulate an error from a node, and make the retry policy return a given * decision for that error. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java index 6ecd6111992..6a7657d5809 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java @@ -37,6 +37,7 @@ import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.core.specex.SpeculativeExecutionPolicy; import com.datastax.oss.driver.api.core.time.TimestampGenerator; +import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.datastax.oss.driver.internal.core.DefaultConsistencyLevelRegistry; import com.datastax.oss.driver.internal.core.ProtocolFeature; @@ -170,7 +171,8 @@ protected RequestHandlerTestHarness(Builder builder) { when(context.getRequestTracker()).thenReturn(new NoopRequestTracker(context)); - when(context.getRequestIdGenerator()).thenReturn(Optional.empty()); + when(context.getRequestIdGenerator()) + .thenReturn(Optional.ofNullable(builder.requestIdGenerator)); } public DefaultSession getSession() { @@ -203,6 +205,7 @@ public static class Builder { private final List poolBehaviors = new ArrayList<>(); private boolean defaultIdempotence; private ProtocolVersion protocolVersion; + private RequestIdGenerator requestIdGenerator; /** * Sets the given node as the next one in the query plan; an empty pool will be simulated when @@ -258,6 +261,11 @@ public Builder withProtocolVersion(ProtocolVersion protocolVersion) { return this; } + public Builder withRequestIdGenerator(RequestIdGenerator requestIdGenerator) { + this.requestIdGenerator = requestIdGenerator; + return this; + } + /** * Sets the given node as the next one in the query plan; the test code is responsible of * calling the methods on the returned object to complete the write and the query. diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java index 2848a8fb629..516a62bb1f7 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/tracker/RequestIdGeneratorIT.java @@ -17,12 +17,14 @@ */ package com.datastax.oss.driver.core.tracker; +import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.core.session.Request; import com.datastax.oss.driver.api.core.tracker.RequestIdGenerator; @@ -119,7 +121,24 @@ public void should_not_write_id_to_custom_payload_when_key_is_not_set() { try (CqlSession session = SessionUtils.newSession(ccmRule, loader)) { String query = "SELECT * FROM system.local"; ResultSet rs = session.execute(query); - assertThat(rs.getExecutionInfo().getRequest().getCustomPayload().get("trace_key")).isNull(); + assertThat(rs.getExecutionInfo().getRequest().getCustomPayload().get("request-id")).isNull(); + } + } + + @Test + public void should_succeed_with_null_value_in_custom_payload() { + DriverConfigLoader loader = + SessionUtils.configLoaderBuilder() + .withString( + DefaultDriverOption.REQUEST_ID_GENERATOR_CLASS, "W3CContextRequestIdGenerator") + .build(); + try (CqlSession session = SessionUtils.newSession(ccmRule, loader)) { + String query = "SELECT * FROM system.local"; + Map customPayload = + new NullAllowingImmutableMap.Builder(1).put("my_key", null).build(); + SimpleStatement statement = + SimpleStatement.newInstance(query).setCustomPayload(customPayload); + assertThatStage(session.executeAsync(statement)).isSuccess(); } } } From 19c60c08b9eecbcfba3c61564bbe15bf3115089a Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 12 Nov 2025 18:31:56 -0600 Subject: [PATCH 71/77] Changelog updates for 4.19.2 patch by Bret McGuire; reviewed by Lukasz Antoniak and Andy Tolbert reference: https://github.com/apache/cassandra-java-driver/pull/2062 --- changelog/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog/README.md b/changelog/README.md index 9e223318e65..b01c3db3bf9 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -21,6 +21,10 @@ under the License. +### 4.19.2 + +- [bug] CASSJAVA-116: Retry or Speculative Execution with RequestIdGenerator throws "Duplicate Key" + ### 4.19.1 - [improvement] CASSJAVA-97: Let users inject an ID for each request and write to the custom payload From 30b05bd7991617b212b8089787fe9ca829d00154 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 13 Nov 2025 12:02:08 -0600 Subject: [PATCH 72/77] [maven-release-plugin] prepare release 4.19.2 --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index fab36abad4c..2e6e476d02a 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-core-shaded - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-mapper-processor - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-mapper-runtime - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-query-builder - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-guava-shaded - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-test-infra - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-metrics-micrometer - 4.19.2-SNAPSHOT + 4.19.2 org.apache.cassandra java-driver-metrics-microprofile - 4.19.2-SNAPSHOT + 4.19.2 com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 617664eec97..b8e56b89b82 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index e750b791d3b..d8a058537f2 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index 27d6d026a4d..71600fbee2c 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index fbf8cd21076..0c2d802266e 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index 498d5dc603a..fc8d3ac0f8e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index df1e2b2613b..29c1aa42001 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.2-SNAPSHOT + 4.19.2 java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index ad581bc0f98..2da3da4024f 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 8360c8a211f..bf122a19e35 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 5368c24c2b6..baa2e42c539 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index a21254da908..5076a146901 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 08211d04c2b..c5eeee518da 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 95c1ca9bb42..1c6359d2cdd 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index abb54e159f6..2fb5f3cb27f 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index 4e3d7006169..c37285e5f7c 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1055,7 +1055,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - HEAD + 4.19.2 diff --git a/query-builder/pom.xml b/query-builder/pom.xml index e1db8485799..a62c800cbd3 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 1a73cde23cd..0f24f638047 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2-SNAPSHOT + 4.19.2 java-driver-test-infra bundle From 62eade21bfeb16a12ce71013fbaebf1c19b5ae96 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 13 Nov 2025 12:02:11 -0600 Subject: [PATCH 73/77] [maven-release-plugin] prepare for next development iteration --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 2e6e476d02a..dd76153a9b1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-core-shaded - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-mapper-processor - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-mapper-runtime - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-query-builder - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-guava-shaded - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-test-infra - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-metrics-micrometer - 4.19.2 + 4.19.3-SNAPSHOT org.apache.cassandra java-driver-metrics-microprofile - 4.19.2 + 4.19.3-SNAPSHOT com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index b8e56b89b82..3727ab9422d 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index d8a058537f2..089e15cd933 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index 71600fbee2c..4c1f11e53a8 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index 0c2d802266e..9cef313f8a5 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index fc8d3ac0f8e..20b9afc1bcd 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index 29c1aa42001..12e42dfdf53 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.2 + 4.19.3-SNAPSHOT java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 2da3da4024f..da2e82e0ab0 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index bf122a19e35..34cb3ef7063 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index baa2e42c539..04d8c98c4f0 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index 5076a146901..57fbd5d3432 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index c5eeee518da..37ba8556a53 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 1c6359d2cdd..9893711d340 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 2fb5f3cb27f..bd3a6380d6b 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index c37285e5f7c..6834cdd1882 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1055,7 +1055,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - 4.19.2 + HEAD diff --git a/query-builder/pom.xml b/query-builder/pom.xml index a62c800cbd3..2bfe1bee8f5 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 0f24f638047..5bf2d07f652 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.2 + 4.19.3-SNAPSHOT java-driver-test-infra bundle From a7da99556b6c202eecc6d5cb1c6371492729ab49 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 4 Dec 2025 12:04:26 -0600 Subject: [PATCH 74/77] Removing interface to Travis CI Patch by Bret McGuire; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/2066 --- .travis.yml | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 84d40ce1356..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -language: java -dist: trusty -sudo: false -# see https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html -matrix: - include: - # 8 - - env: JDK='OpenJDK 8' - jdk: openjdk8 - # 11 - - env: JDK='OpenJDK 11' - # switch to JDK 11 before running tests - before_script: . $TRAVIS_BUILD_DIR/ci/install-jdk.sh -F 11 -L GPL -before_install: - # Require JDK8 for compiling - - jdk_switcher use openjdk8 - - ./install-snapshots.sh -install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V -script: mvn test -Djacoco.skip=true -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true -B -V -cache: - directories: - - $HOME/.m2 From ded2985f14e9774cb46dd0104b1f045613bc0279 Mon Sep 17 00:00:00 2001 From: "April I. Murphy" <36110273+aimurphy@users.noreply.github.com> Date: Thu, 20 Nov 2025 08:21:36 -0800 Subject: [PATCH 75/77] Replace outdated link in README patch by April Murphy; reviewed by Bret McGuire and Lukasz Antoniak reference: https://github.com/apache/cassandra-java-driver/pull/2064 --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 0f6c2bb5a6f..5994a4be8a1 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,6 @@ remain unchanged, and the new API will look very familiar to 2.x and 3.x users. See the [upgrade guide](upgrade_guide/) for details. -## Error Handling - -See the [Cassandra error handling done right blog](https://www.datastax.com/blog/cassandra-error-handling-done-right) for error handling with the Java Driver for Apache Cassandra™. - ## Useful links * [Manual](manual/) From e762df872b7ca02b4cd5cf780bd96f77815c9646 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 19 Dec 2025 13:13:18 +0100 Subject: [PATCH 76/77] ninja-fix: Remove ASF donation message --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5994a4be8a1..d8ef01d0964 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Java Driver for Apache Cassandra® -:warning: The java-driver has recently been donated by Datastax to The Apache Software Foundation and the Apache Cassandra project. Bear with us as we move assets and coordinates. - [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core) From 595cb29912dc8b55663cc13bafe3f17dc4f91ce6 Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Mon, 22 Dec 2025 22:20:20 -0800 Subject: [PATCH 77/77] Update LZ4 and Netty dependencies for CVE response The primary goal here is to address CVE-2025-12183. Netty includes a dependency on vulnerable versions of lz4-java, so update to a fixed version of Netty as well. On the C* server side, we opted to move to the new community fork of lz4-java, so match that decision here (CASSANDRA-21052). patch by Abe Ratnofsky; reviewed by Francisco Guerrero for CASSJAVA-113 --- NOTICE_binary.txt | 2 +- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- core/src/main/resources/reference.conf | 2 +- .../internal/core/insights/PlatformInfoFinderTest.java | 2 +- core/src/test/resources/insights/test-dependencies.txt | 2 +- integration-tests/pom.xml | 2 +- manual/core/compression/README.md | 6 +++--- manual/core/integration/README.md | 2 +- osgi-tests/pom.xml | 2 +- .../oss/driver/internal/osgi/support/BundleOptions.java | 2 +- pom.xml | 6 +++--- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/NOTICE_binary.txt b/NOTICE_binary.txt index c60d8ceb245..f6f11c298f6 100644 --- a/NOTICE_binary.txt +++ b/NOTICE_binary.txt @@ -100,7 +100,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * license/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java This product optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 3727ab9422d..84cb4b15398 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -74,7 +74,7 @@ true - org.lz4 + at.yawk.lz4 lz4-java true diff --git a/core/pom.xml b/core/pom.xml index 089e15cd933..8758d20d78a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -73,7 +73,7 @@ true - org.lz4 + at.yawk.lz4 lz4-java true diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 741b1d97654..4ae83362e29 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1114,7 +1114,7 @@ datastax-java-driver { # The name of the algorithm used to compress protocol frames. # # The possible values are: - # - lz4: requires net.jpountz.lz4:lz4 in the classpath. + # - lz4: requires at.yawk.lz4:lz4-java in the classpath. # - snappy: requires org.xerial.snappy:snappy-java in the classpath. # - the string "none" to indicate no compression (this is functionally equivalent to omitting # the option). diff --git a/core/src/test/java/com/datastax/dse/driver/internal/core/insights/PlatformInfoFinderTest.java b/core/src/test/java/com/datastax/dse/driver/internal/core/insights/PlatformInfoFinderTest.java index 80294ea6b7d..2a098363d46 100644 --- a/core/src/test/java/com/datastax/dse/driver/internal/core/insights/PlatformInfoFinderTest.java +++ b/core/src/test/java/com/datastax/dse/driver/internal/core/insights/PlatformInfoFinderTest.java @@ -77,7 +77,7 @@ public void should_find_dependencies_from_file() { "com.fasterxml.jackson.core:jackson-annotations", withUnverifiedRuntimeVersion("2.8.11")); expected.put("com.fasterxml.jackson.core:jackson-core", withUnverifiedRuntimeVersion("2.8.11")); expected.put("io.netty:netty-handler", withUnverifiedRuntimeVersion("4.0.56.Final")); - expected.put("org.lz4:lz4-java", withUnverifiedRuntimeVersionOptional("1.4.1")); + expected.put("at.yawk.lz4:lz4-java", withUnverifiedRuntimeVersionOptional("1.10.1")); expected.put("org.hdrhistogram:HdrHistogram", withUnverifiedRuntimeVersionOptional("2.1.10")); expected.put("com.github.jnr:jffi", withUnverifiedRuntimeVersion("1.2.16")); expected.put("io.netty:netty-buffer", withUnverifiedRuntimeVersion("4.0.56.Final")); diff --git a/core/src/test/resources/insights/test-dependencies.txt b/core/src/test/resources/insights/test-dependencies.txt index 6cabe8b257d..e9186a35e6b 100644 --- a/core/src/test/resources/insights/test-dependencies.txt +++ b/core/src/test/resources/insights/test-dependencies.txt @@ -17,7 +17,7 @@ The following files have been resolved: com.fasterxml.jackson.core:jackson-core:jar:2.8.11:compile org.hdrhistogram:HdrHistogram:jar:2.1.10:compile (optional) org.ow2.asm:asm-tree:jar:5.0.3:compile - org.lz4:lz4-java:jar:1.4.1:compile (optional) + at.yawk.lz4:lz4-java:jar:1.10.1:compile (optional) io.netty:netty-transport:jar:4.0.56.Final:compile io.dropwizard.metrics:metrics-core:jar:3.2.2:compile io.netty:netty-common:jar:4.0.56.Final:compile diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 34cb3ef7063..e302e12077f 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -129,7 +129,7 @@ test - org.lz4 + at.yawk.lz4 lz4-java test diff --git a/manual/core/compression/README.md b/manual/core/compression/README.md index 9e84fde917d..9f7ae3c4854 100644 --- a/manual/core/compression/README.md +++ b/manual/core/compression/README.md @@ -46,7 +46,7 @@ datastax-java-driver { Compression must be set before opening a session, it cannot be changed at runtime. -Two algorithms are supported out of the box: [LZ4](https://github.com/jpountz/lz4-java) and +Two algorithms are supported out of the box: [LZ4](https://github.com/yawkat/lz4-java) and [Snappy](http://google.github.io/snappy/). The LZ4 implementation is a good first choice; it offers fallback implementations in case native libraries fail to load and [benchmarks](http://java-performance.info/performance-general-compression/) suggest that it offers @@ -63,9 +63,9 @@ Dependency: ```xml - org.lz4 + at.yawk.lz4 lz4-java - 1.4.1 + 1.10.1 ``` diff --git a/manual/core/integration/README.md b/manual/core/integration/README.md index f2a96160bce..e2c7bc218ee 100644 --- a/manual/core/integration/README.md +++ b/manual/core/integration/README.md @@ -416,7 +416,7 @@ are not available on your platform, you can exclude the following dependency: #### Compression libraries -The driver supports compression with either [LZ4](https://github.com/jpountz/lz4-java) or +The driver supports compression with either [LZ4](https://github.com/yawkat/lz4-java) or [Snappy](http://google.github.io/snappy/). These dependencies are optional; you have to add them explicitly in your application in order to diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index bd3a6380d6b..c2cc4d830f1 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -79,7 +79,7 @@ snappy-java - org.lz4 + at.yawk.lz4 lz4-java diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java index 3e6171ca530..378b515aa65 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java @@ -117,7 +117,7 @@ public static CompositeOption jacksonBundles() { public static CompositeOption lz4Bundle() { return () -> options( - mavenBundle("org.lz4", "lz4-java").versionAsInProject(), + mavenBundle("at.yawk.lz4", "lz4-java").versionAsInProject(), systemProperty("cassandra.compression").value("LZ4")); } diff --git a/pom.xml b/pom.xml index 6834cdd1882..eb83459cfb4 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 2.1.12 4.1.18 - 4.1.119.Final + 4.1.130.Final 1.2.1 1.1.10.1 - 1.7.1 + 1.10.1 3.19.0 1.3 @@ -137,7 +137,7 @@ ${snappy.version} - org.lz4 + at.yawk.lz4 lz4-java ${lz4.version}