gradle · asodja · Feb 6, 2023 · Feb 6, 2023 · Feb 6, 2023 · Feb 6, 2023
diff --git a/...g/gradle/performance/regression/buildcache/AbstractTaskOutputCachingCrossBuildTest.groovy b/...g/gradle/performance/regression/buildcache/AbstractTaskOutputCachingCrossBuildTest.groovy
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.performance.regression.buildcache
+
+
+import org.gradle.performance.AbstractCrossBuildPerformanceTest
+import org.gradle.profiler.BuildContext
+import org.gradle.profiler.BuildMutator
+import org.gradle.profiler.InvocationSettings
+import org.gradle.profiler.Phase
+import org.gradle.test.fixtures.file.TestFile
+import org.gradle.test.fixtures.server.http.HttpBuildCacheServer
+import org.junit.Rule
+
+class AbstractTaskOutputCachingCrossBuildTest extends AbstractCrossBuildPerformanceTest {
+    int firstWarmupWithCache = 1
+    TestFile cacheDir
+    String protocol = "http"
+    boolean pushToRemote
+    boolean checkIfCacheUsed = true
+
+    @Rule
+    HttpBuildCacheServer buildCacheServer = new HttpBuildCacheServer(temporaryFolder)
+
+    def setup() {
+        buildCacheServer.logRequests = false
+        cacheDir = temporaryFolder.file("local-cache")
+        runner.addBuildMutator { invocationSettings ->
+            new BuildMutator() {
+                @Override
+                void beforeBuild(BuildContext context) {
+                    if (isRunWithCache(context)) {
+                        if (!buildCacheServer.isRunning()) {
+                            buildCacheServer.start()
+                        }
+                        def settings = new TestFile(invocationSettings.projectDir).file('settings.gradle')
+                        if (isFirstRunWithCache(context)) {
+                            cacheDir.deleteDir().mkdirs()
+                            buildCacheServer.cacheDir.deleteDir().mkdirs()
+                            settings << remoteCacheSettingsScript
+                        }
+                        assert buildCacheServer.uri.toString().startsWith("${protocol}://")
+                        assert settings.text.contains(buildCacheServer.uri.toString())
+                    }
+                }
+
+                @Override
+                void afterBuild(BuildContext context, Throwable error) {
+                    if (isLastRun(context, invocationSettings) && checkIfCacheUsed) {
+                        assert !(buildCacheServer.cacheDir.allDescendants().empty && cacheDir.allDescendants().isEmpty())
+                        assert pushToRemote || buildCacheServer.cacheDir.allDescendants().empty
+                    }
+                }
+            }
+        }
+    }
+
+    static boolean isLastRun(BuildContext context, InvocationSettings invocationSettings) {
+        context.iteration == invocationSettings.buildCount && context.phase == Phase.MEASURE
+    }
+
+    boolean isRunWithCache(BuildContext context) {
+        context.iteration >= firstWarmupWithCache || context.phase == Phase.MEASURE
+    }
+
+    boolean isFirstRunWithCache(BuildContext context) {
+        context.iteration == firstWarmupWithCache && context.phase == Phase.WARM_UP
+    }
+
+    String getRemoteCacheSettingsScript() {
+        """
+            def httpCacheClass = Class.forName('org.gradle.caching.http.HttpBuildCache')
+            buildCache {
+                local {
+                    directory = '${cacheDir.absoluteFile.toURI()}'
+                }
+                remote(httpCacheClass) {
+                    url = '${buildCacheServer.uri}/'
+                    push = ${pushToRemote}
+                }
+            }
+        """.stripIndent()
+    }
+
+    BuildMutator cleanLocalCache() {
+        new BuildMutator() {
+            @Override
+            void beforeBuild(BuildContext context) {
+                cacheDir.deleteDir().mkdirs()
+            }
+        }
+    }
+
+    BuildMutator cleanRemoteCache() {
+        new BuildMutator() {
+            @Override
+            void beforeBuild(BuildContext context) {
+                buildCacheServer.cacheDir.deleteDir().mkdirs()
+            }
+        }
+    }
+}
diff --git a/...NextGenerationBuildCacheVsProductionBuildCacheTaskOutputCachingJavaPerformanceTest.groovy b/...NextGenerationBuildCacheVsProductionBuildCacheTaskOutputCachingJavaPerformanceTest.groovy
@@ -0,0 +1,250 @@
+/*
+ * Copyright 2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.performance.regression.buildcache
+
+import org.gradle.initialization.StartParameterBuildOptions
+import org.gradle.performance.annotations.RunFor
+import org.gradle.performance.annotations.Scenario
+import org.gradle.performance.fixture.CrossBuildPerformanceTestRunner
+import org.gradle.performance.fixture.GradleBuildExperimentSpec
+import org.gradle.performance.fixture.JavaTestProject
+import org.gradle.performance.results.BaselineVersion
+import org.gradle.performance.results.CrossBuildPerformanceResults
+import org.gradle.profiler.mutations.ApplyAbiChangeToJavaSourceFileMutator
+import org.gradle.profiler.mutations.ApplyNonAbiChangeToJavaSourceFileMutator
+import org.gradle.test.fixtures.keystore.TestKeyStore
+
+import java.util.function.Consumer
+
+import static org.gradle.performance.annotations.ScenarioType.PER_COMMIT
+import static org.gradle.performance.annotations.ScenarioType.PER_DAY
+import static org.gradle.performance.results.OperatingSystem.LINUX
+import static org.gradle.performance.results.PerformanceTestResult.hasRegressionChecks
+/**
+ * This test tests Next generation build cache vs production build cache. The version we test will always use next generation build cache
+ * and baseline will use production build cache.
+ */
+@RunFor(
+    @Scenario(type = PER_DAY, operatingSystems = [LINUX], testProjects = ["largeJavaMultiProject", "largeMonolithicJavaProject"])
+)
+class NextGenerationBuildCacheVsProductionBuildCacheTaskOutputCachingJavaPerformanceTest extends AbstractTaskOutputCachingCrossBuildTest {
+
+    private static final String NG_BUILD_CACHE_DISPLAY_NAME = "Next generation build cache"
+    private static final String PRODUCTION_BUILD_CACHE_DISPLAY_NAME = "Production build cache"
+
+    boolean callClean
+
+    def setup() {
+        callClean = true
+        runner.testGroup = "Build cache NG"
+    }
+
+    def "clean assemble with remote http cache"() {
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.warmUpCount = 2
+            builder.invocationCount = 8
+            builder.invocation.useDaemon = false
+            builder.addBuildMutator { cleanLocalCache() }
+        }
+        pushToRemote = true
+        protocol = "http"
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    def "clean assemble with remote https cache"() {
+        def keyStore = TestKeyStore.init(temporaryFolder.file('ssl-keystore'))
+        keyStore.enableSslWithServerCert(buildCacheServer)
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.warmUpCount = 2
+            builder.invocationCount = 8
+            builder.invocation.useDaemon = false
+            builder.addBuildMutator { cleanLocalCache() }
+            builder.invocation.args.addAll(keyStore.serverAndClientCertArgs)
+        }
+        protocol = "https"
+        pushToRemote = true
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    def "clean assemble with empty local cache"() {
+        given:
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.warmUpCount = 2
+            builder.invocationCount = 8
+            builder.invocation.useDaemon = false
+            builder.addBuildMutator { cleanLocalCache() }
+        }
+        pushToRemote = false
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    def "clean assemble with empty remote http cache"() {
+        given:
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.warmUpCount = 2
+            builder.invocationCount = 8
+            builder.invocation.useDaemon = false
+            builder.addBuildMutator { cleanLocalCache() }
+            builder.addBuildMutator { cleanRemoteCache() }
+        }
+        pushToRemote = true
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    @RunFor(
+        @Scenario(type = PER_COMMIT, operatingSystems = [LINUX], testProjects = ["largeJavaMultiProject", "largeMonolithicJavaProject"])
+    )
+    def "clean assemble with local cache"() {
+        given:
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.invocation.args += "--parallel"
+        }
+        pushToRemote = false
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    @RunFor([
+        @Scenario(type = PER_COMMIT, operatingSystems = [LINUX], testProjects = ["largeJavaMultiProject"],
+            comment = "We only test the multi-project here since for the monolithic project we would have no cache hits. This would mean we actually would test incremental compilation."
+        )
+    ])
+    def "clean assemble for abi change with local cache"() {
+        given:
+        def testProject = JavaTestProject.projectFor(runner.testProject)
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.addBuildMutator { new ApplyAbiChangeToJavaSourceFileMutator(new File(it.projectDir, testProject.config.fileToChangeByScenario['assemble'])) }
+            builder.invocation.args += "--parallel"
+        }
+        pushToRemote = false
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    @RunFor([
+        @Scenario(type = PER_COMMIT, operatingSystems = [LINUX], testProjects = ["largeJavaMultiProject"],
+            comment = "We only test the multi-project here since for the monolithic project we would have no cache hits. This would mean we actually would test incremental compilation."
+        )
+    ])
+    def "clean assemble for non-abi change with local cache"() {
+        given:
+        def testProject = JavaTestProject.projectFor(runner.testProject)
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.addBuildMutator { new ApplyNonAbiChangeToJavaSourceFileMutator(new File(it.projectDir, testProject.config.fileToChangeByScenario['assemble'])) }
+            builder.invocation.args += "--parallel"
+        }
+        pushToRemote = false
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    @RunFor([
+        @Scenario(type = PER_COMMIT, operatingSystems = [LINUX], testProjects = ["largeJavaMultiProject"],
+            comment = "We only test the multi-project here since for the monolithic project we would have no cache hits. This would mean we actually would test incremental compilation."
+        )
+    ])
+    def "no-clean assemble for non-abi change with local cache"() {
+        given:
+        callClean = false
+        def testProject = JavaTestProject.projectFor(runner.testProject)
+        setupTestProject(runner) { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.addBuildMutator { new ApplyNonAbiChangeToJavaSourceFileMutator(new File(it.projectDir, testProject.config.fileToChangeByScenario['assemble'])) }
+            builder.invocation.args += "--parallel"
+        }
+        pushToRemote = false
+
+        when:
+        def result = runner.run()
+
+        then:
+        assertNotRegressed(result)
+    }
+
+    def setupTestProject(CrossBuildPerformanceTestRunner runner, Consumer<GradleBuildExperimentSpec.GradleBuilder> configure) {
+        Consumer<GradleBuildExperimentSpec.GradleBuilder> defaultConfiguration = { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.invocation {
+                tasksToRun("assemble")
+                if (callClean) {
+                    cleanTasks("clean")
+                }
+                args("-D${StartParameterBuildOptions.BuildCacheOption.GRADLE_PROPERTY}=true")
+            }
+            builder.warmUpCount = 11
+            builder.invocationCount = 21
+        }
+        runner.buildSpec { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.displayName(NG_BUILD_CACHE_DISPLAY_NAME)
+            builder.invocation {
+                args("-Dorg.gradle.unsafe.cache.ng=true")
+            }
+            defaultConfiguration.accept(builder)
+            configure.accept(builder)
+        }
+        runner.baseline { GradleBuildExperimentSpec.GradleBuilder builder ->
+            builder.displayName(PRODUCTION_BUILD_CACHE_DISPLAY_NAME)
+            defaultConfiguration.accept(builder)
+            configure.accept(builder)
+        }
+    }
+
+    static def assertNotRegressed(CrossBuildPerformanceResults result) {
+        def nextGeneration = result.buildResult(NG_BUILD_CACHE_DISPLAY_NAME)
+        def production = result.buildResult(PRODUCTION_BUILD_CACHE_DISPLAY_NAME)
+
+        def productionResults = new BaselineVersion("Production")
+        productionResults.results.addAll(production)
+        def stats = productionResults.getSpeedStatsAgainst("Next-Generation", nextGeneration)
+        println(stats)
+        boolean isProductionFaster = productionResults.significantlyFasterThan(nextGeneration)
+        if (isProductionFaster && hasRegressionChecks()) {
+            throw new AssertionError(stats as Object)
+        }
+        return result
+    }
+}