Fix remote build cache (#2298)

Author: nedtwigg

CHANGES.md

@@ -13,6 +13,7 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
 ### Added
 * Support for `rdf` ([#2261](https://github.com/diffplug/spotless/pull/2261))
 * Support for `buf` on maven plugin ([#2291](https://github.com/diffplug/spotless/pull/2291))
+* `ConfigurationCacheHack` so we can support Gradle's configuration cache and remote build cache at the same time. ([TODO]()fixes [#2168](https://github.com/diffplug/spotless/issues/2168))
 ### Changed
 * Support configuring the Equo P2 cache. ([#2238](https://github.com/diffplug/spotless/pull/2238))
 * Add explicit support for JSONC / CSS via biome, via the file extensions `.css` and `.jsonc`. 

lib/src/main/java/com/diffplug/spotless/ConfigurationCacheHackList.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2024 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.spotless;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Gradle requires three things:
+ * - Gradle defines cache equality based on your serialized representation
+ * - Combined with remote build cache, you cannot have any absolute paths in
+ * your serialized representation
+ * - Combined with configuration cache, you must be able to roundtrip yourself
+ * through serialization
+ *
+ * These requirements are at odds with each other, as described in these issues
+ * - Gradle issue to define custom equality
+ * https://github.com/gradle/gradle/issues/29816
+ * - Spotless plea for developer cache instead of configuration cache
+ * https://github.com/diffplug/spotless/issues/987
+ * - Spotless cache miss bug fixed by this class
+ * https://github.com/diffplug/spotless/issues/2168
+ *
+ * This class is a `List<FormatterStep>` which can optimize the
+ * serialized representation for either
+ * - roundtrip integrity
+ * - OR
+ * - equality
+ *
+ * Because it is not possible to provide both at the same time.
+ * It is a horrific hack, but it works, and it's the only way I can figure
+ * to make Spotless work with all of Gradle's cache systems at once.
+ */
+public class ConfigurationCacheHackList implements java.io.Serializable {
+	private static final long serialVersionUID = 1L;
+	private final boolean optimizeForEquality;
+	private final ArrayList<Object> backingList = new ArrayList<>();
+
+	public static ConfigurationCacheHackList forEquality() {
+		return new ConfigurationCacheHackList(true);
+	}
+
+	public static ConfigurationCacheHackList forRoundtrip() {
+		return new ConfigurationCacheHackList(false);
+	}
+
+	private ConfigurationCacheHackList(boolean optimizeForEquality) {
+		this.optimizeForEquality = optimizeForEquality;
+	}
+
+	public void clear() {
+		backingList.clear();
+	}
+
+	public void addAll(Collection<? extends FormatterStep> c) {
+		for (FormatterStep step : c) {
+			if (step instanceof FormatterStepSerializationRoundtrip) {
+				var clone = ((FormatterStepSerializationRoundtrip) step).hackClone(optimizeForEquality);
+				backingList.add(clone);
+			} else {
+				backingList.add(step);
+			}
+		}
+	}
+
+	public List<FormatterStep> getSteps() {
+		var result = new ArrayList<FormatterStep>(backingList.size());
+		for (Object obj : backingList) {
+			if (obj instanceof FormatterStepSerializationRoundtrip.HackClone) {
+				result.add(((FormatterStepSerializationRoundtrip.HackClone) obj).rehydrate());
+			} else {
+				result.add((FormatterStep) obj);
+			}
+		}
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ConfigurationCacheHackList stepList = (ConfigurationCacheHackList) o;
+		return optimizeForEquality == stepList.optimizeForEquality &&
+				backingList.equals(stepList.backingList);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(optimizeForEquality, backingList);
+	}
+}

lib/src/main/java/com/diffplug/spotless/FormatterStepSerializationRoundtrip.java

@@ -16,20 +16,23 @@
 package com.diffplug.spotless;
 
 import java.io.IOException;
-import java.io.ObjectStreamException;
 import java.io.Serializable;
+import java.util.Objects;
 
 import edu.umd.cs.findbugs.annotations.Nullable;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
-class FormatterStepSerializationRoundtrip<RoundtripState extends Serializable, EqualityState extends Serializable> extends FormatterStepEqualityOnStateSerialization<EqualityState> {
+final class FormatterStepSerializationRoundtrip<RoundtripState extends Serializable, EqualityState extends Serializable> extends FormatterStepEqualityOnStateSerialization<EqualityState> {
 	private static final long serialVersionUID = 1L;
 	private final String name;
+	@SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "HackClone")
 	private final transient ThrowingEx.Supplier<RoundtripState> initializer;
 	private @Nullable RoundtripState roundtripStateInternal;
+	private @Nullable EqualityState equalityStateInternal;
 	private final SerializedFunction<RoundtripState, EqualityState> equalityStateExtractor;
 	private final SerializedFunction<EqualityState, FormatterFunc> equalityStateToFormatter;
 
-	public FormatterStepSerializationRoundtrip(String name, ThrowingEx.Supplier<RoundtripState> initializer, SerializedFunction<RoundtripState, EqualityState> equalityStateExtractor, SerializedFunction<EqualityState, FormatterFunc> equalityStateToFormatter) {
+	FormatterStepSerializationRoundtrip(String name, ThrowingEx.Supplier<RoundtripState> initializer, SerializedFunction<RoundtripState, EqualityState> equalityStateExtractor, SerializedFunction<EqualityState, FormatterFunc> equalityStateToFormatter) {
 		this.name = name;
 		this.initializer = initializer;
 		this.equalityStateExtractor = equalityStateExtractor;
@@ -41,32 +44,97 @@ public String getName() {
 		return name;
 	}
 
-	@Override
-	protected EqualityState stateSupplier() throws Exception {
+	private RoundtripState roundtripStateSupplier() throws Exception {
 		if (roundtripStateInternal == null) {
 			roundtripStateInternal = initializer.get();
 		}
-		return equalityStateExtractor.apply(roundtripStateInternal);
+		return roundtripStateInternal;
+	}
+
+	@Override
+	protected EqualityState stateSupplier() throws Exception {
+		if (equalityStateInternal == null) {
+			equalityStateInternal = equalityStateExtractor.apply(roundtripStateSupplier());
+		}
+		return equalityStateInternal;
 	}
 
 	@Override
 	protected FormatterFunc stateToFormatter(EqualityState equalityState) throws Exception {
 		return equalityStateToFormatter.apply(equalityState);
 	}
 
-	// override serialize output
 	private void writeObject(java.io.ObjectOutputStream out) throws IOException {
-		if (roundtripStateInternal == null) {
-			roundtripStateInternal = ThrowingEx.get(initializer::get);
+		if (initializer == null) {
+			// then this instance was created by Gradle's ConfigurationCacheHackList and the following will hold true
+			if (roundtripStateInternal == null && equalityStateInternal == null) {
+				throw new IllegalStateException("If the initializer was null, then one of roundtripStateInternal or equalityStateInternal should be non-null, and neither was");
+			}
+		} else {
+			// this was a normal instance, which means we need to encode to roundtripStateInternal (since the initializer might not be serializable)
+			// and there's no reason to keep equalityStateInternal since we can always recompute it
+			if (roundtripStateInternal == null) {
+				roundtripStateInternal = ThrowingEx.get(this::roundtripStateSupplier);
+			}
+			equalityStateInternal = null;
 		}
 		out.defaultWriteObject();
 	}
 
-	private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
-		in.defaultReadObject();
+	HackClone<?, ?> hackClone(boolean optimizeForEquality) {
+		return new HackClone<>(this, optimizeForEquality);
 	}
 
-	private void readObjectNoData() throws ObjectStreamException {
-		throw new UnsupportedOperationException();
+	/**
+	 * This class has one setting (optimizeForEquality) and two pieces of data
+	 * - the original step, which is marked transient so it gets discarded during serialization
+	 * - the cleaned step, which is lazily created during serialization, and the serialized form is optimized for either equality or roundtrip integrity
+	 *
+	 * It works in conjunction with ConfigurationCacheHackList to allow Spotless to work with all of Gradle's cache systems.
+	 */
+	static class HackClone<RoundtripState extends Serializable, EqualityState extends Serializable> implements Serializable {
+		private static final long serialVersionUID = 1L;
+		@SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "HackClone")
+		transient FormatterStepSerializationRoundtrip<?, ?> original;
+		boolean optimizeForEquality;
+		@Nullable
+		FormatterStepSerializationRoundtrip cleaned;
+
+		HackClone(@Nullable FormatterStepSerializationRoundtrip<RoundtripState, EqualityState> original, boolean optimizeForEquality) {
+			this.original = original;
+			this.optimizeForEquality = optimizeForEquality;
+		}
+
+		@SuppressFBWarnings(value = "NP_NONNULL_PARAM_VIOLATION", justification = "HackClone")
+		private void writeObject(java.io.ObjectOutputStream out) throws IOException {
+			if (cleaned == null) {
+				cleaned = new FormatterStepSerializationRoundtrip(original.name, null, original.equalityStateExtractor, original.equalityStateToFormatter);
+				if (optimizeForEquality) {
+					cleaned.equalityStateInternal = ThrowingEx.get(original::stateSupplier);
+				} else {
+					cleaned.roundtripStateInternal = ThrowingEx.get(original::roundtripStateSupplier);
+				}
+			}
+			out.defaultWriteObject();
+		}
+
+		public FormatterStep rehydrate() {
+			return original != null ? original : Objects.requireNonNull(cleaned, "how is clean null if this has been serialized?");
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o)
+				return true;
+			if (o == null || getClass() != o.getClass())
+				return false;
+			HackClone<?, ?> that = (HackClone<?, ?>) o;
+			return optimizeForEquality == that.optimizeForEquality && rehydrate().equals(that.rehydrate());
+		}
+
+		@Override
+		public int hashCode() {
+			return rehydrate().hashCode() ^ Boolean.hashCode(optimizeForEquality);
+		}
 	}
 }

plugin-gradle/CHANGES.md

@@ -17,6 +17,7 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
 * Bump default `cleanthat` version to latest `2.21` -> `2.22`. ([#2296](https://github.com/diffplug/spotless/pull/2296))
 ### Fixed
 * Java import order, ignore duplicate group entries. ([#2293](https://github.com/diffplug/spotless/pull/2293))
+* Remote build cache shouldn't have cache misses anymore. ([TODO]()fixes [#2168](https://github.com/diffplug/spotless/issues/2168))
 
 ## [7.0.0.BETA2] - 2024-08-25
 ### Changed

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/SpotlessTask.java

@@ -17,8 +17,6 @@
 
 import java.io.File;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
@@ -37,6 +35,7 @@
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.work.Incremental;
 
+import com.diffplug.spotless.ConfigurationCacheHackList;
 import com.diffplug.spotless.FormatExceptionPolicy;
 import com.diffplug.spotless.FormatExceptionPolicyStrict;
 import com.diffplug.spotless.Formatter;
@@ -150,17 +149,25 @@ public File getOutputDirectory() {
 		return outputDirectory;
 	}
 
-	protected final List<FormatterStep> steps = new ArrayList<>();
+	private final ConfigurationCacheHackList stepsInternalRoundtrip = ConfigurationCacheHackList.forRoundtrip();
+	private final ConfigurationCacheHackList stepsInternalEquality = ConfigurationCacheHackList.forEquality();
+
+	@Internal
+	public ConfigurationCacheHackList getStepsInternalRoundtrip() {
+		return stepsInternalRoundtrip;
+	}
 
 	@Input
-	public List<FormatterStep> getSteps() {
-		return Collections.unmodifiableList(steps);
+	public ConfigurationCacheHackList getStepsInternalEquality() {
+		return stepsInternalEquality;
 	}
 
 	public void setSteps(List<FormatterStep> steps) {
 		PluginGradlePreconditions.requireElementsNonNull(steps);
-		this.steps.clear();
-		this.steps.addAll(steps);
+		this.stepsInternalRoundtrip.clear();
+		this.stepsInternalEquality.clear();
+		this.stepsInternalRoundtrip.addAll(steps);
+		this.stepsInternalEquality.addAll(steps);
 	}
 
 	/** Returns the name of this format. */
@@ -179,7 +186,7 @@ Formatter buildFormatter() {
 				.lineEndingsPolicy(getLineEndingsPolicy().get())
 				.encoding(Charset.forName(encoding))
 				.rootDir(getProjectDir().get().getAsFile().toPath())
-				.steps(steps)
+				.steps(stepsInternalRoundtrip.getSteps())
 				.exceptionPolicy(exceptionPolicy)
 				.build();
 	}