Migrate spotless, muzzle, and test-published-artifacts from CircleCI to Gitlab (#8385)

* Add checks, spotless, and test_published_artifact

* reduce to jobs we care about

* remove from circleci config

* remove spotless as well

* add "required" job

* fix image tag for fan-in

* simplify

* only relock during scheduled nightly

Author: randomanderson

.circleci/config.continue.yml.j2

@@ -365,21 +365,6 @@ jobs:
 
       - display_memory_usage
 
-  spotless:
-    <<: *defaults
-    resource_class: medium+
-
-    steps:
-      - setup_code
-
-      - run:
-          name: Run spotless
-          command: >-
-            JAVA_HOME=$JAVA_11_HOME
-            ./gradlew spotlessCheck
-            << pipeline.parameters.gradle_flags >>
-            --max-workers=8
-
   check:
     <<: *defaults
 
@@ -705,117 +690,6 @@ jobs:
           - DD_BIND_HOST=0.0.0.0
           - DD_API_KEY=invalid_key_but_this_is_fine
 
-  test_published_artifacts:
-    <<: *defaults
-    resource_class: medium
-    docker:
-      - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}7
-
-    steps:
-      - setup_code
-      - restore_dependency_cache:
-          cacheType: lib
-      - restore_build_cache:
-          cacheType: lib
-
-      - run:
-          name: Publish Artifacts Locally
-          command: |
-            mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
-            rm -rf "${mvn_local_repo}/com/datadoghq"
-            export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-            ./gradlew publishToMavenLocal << pipeline.parameters.gradle_flags >> --max-workers=3
-
-      - run:
-          name: Test Published Artifacts
-          command: |
-            cd test-published-dependencies
-            export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-            ./gradlew check --info --max-workers=3
-
-      - run:
-          name: Collect Reports
-          when: on_fail
-          command: .circleci/collect_reports.sh
-
-      - store_artifacts:
-          path: ./reports
-
-      - display_memory_usage
-  muzzle-dep-report:
-    <<: *defaults
-    resource_class: medium
-    steps:
-      - setup_code
-      - skip_unless_matching_files_changed:
-          pattern: "dd-java-agent/instrumentation"
-      - restore_dependency_cache:
-          cacheType: inst
-      - restore_build_cache:
-          cacheType: inst
-      - run:
-          name: Generate muzzle dep report
-          command: >-
-            SKIP_BUILDSCAN="true"
-            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-            ./gradlew generateMuzzleReport muzzleInstrumentationReport
-      - run:
-          name: Collect Reports
-          command: .circleci/collect_muzzle_deps.sh
-      - store_artifacts:
-          path: ./reports
-
-  muzzle:
-    <<: *defaults
-    resource_class: medium+
-    parallelism: 4
-    steps:
-      - setup_code
-
-      - skip_unless_matching_files_changed:
-          pattern: "dd-java-agent/instrumentation"
-
-      # We are not running with a separate cache of all muzzle artifacts here because it gets very big and
-      # ends up taking more time restoring/saving than the actual increase in time it takes just
-      # downloading the artifacts each time.
-      #
-      # Let's at least restore the build cache to have something to start from.
-      - restore_dependency_cache:
-          cacheType: inst
-      - restore_build_cache:
-          cacheType: inst
-
-      - run:
-          name: Gather muzzle tasks
-          command: >-
-            SKIP_BUILDSCAN="true"
-            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-            ./gradlew writeMuzzleTasksToFile
-            << pipeline.parameters.gradle_flags >>
-            --max-workers=3
-
-      - run:
-          name: Verify Muzzle
-          command: >-
-            SKIP_BUILDSCAN="true"
-            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx3G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-            ./gradlew `circleci tests split --split-by=timings workspace/build/muzzleTasks | xargs`
-            << pipeline.parameters.gradle_flags >>
-            --max-workers=4
-
-      - run:
-          name: Collect Reports
-          when: on_fail
-          command: .circleci/collect_reports.sh
-
-      - store_artifacts:
-          path: ./reports
-
-      - store_test_results:
-          path: workspace/build/muzzle-test-results
-
-      - display_memory_usage
-
   system-tests:
     machine:
       image: ubuntu-2404:current
@@ -1069,7 +943,6 @@ build_test_jobs: &build_test_jobs
       name: build_profiling
       gradleTarget: :profilingTest
       cacheType: profiling
-  - spotless
 
   - fan_in:
       requires:
@@ -1078,7 +951,6 @@ build_test_jobs: &build_test_jobs
         - build_inst
         - build_smoke
         - build_profiling
-        - spotless
       name: ok_to_test
       stage: ok_to_test
 
@@ -1427,24 +1299,6 @@ build_test_jobs: &build_test_jobs
       cacheType: base
       testJvm: "8"
 
-  - test_published_artifacts:
-      requires:
-        - ok_to_test
-
-  - muzzle:
-      requires:
-        - ok_to_test
-      filters:
-        branches:
-          ignore:
-            - master
-            - project/*
-            - release/*
-
-  - muzzle-dep-report:
-      requires:
-        - ok_to_test
-
   - system-tests:
       requires:
         - ok_to_test
@@ -1465,7 +1319,6 @@ build_test_jobs: &build_test_jobs
 
   - fan_in:
       requires:
-        - test_published_artifacts
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}_profiling"
 {% endfor %}
@@ -1474,7 +1327,6 @@ build_test_jobs: &build_test_jobs
 
   - fan_in:
       requires:
-        - test_published_artifacts
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}_debugger"
 {% endfor %}
@@ -1486,13 +1338,11 @@ build_test_jobs: &build_test_jobs
   - fan_in:
       requires:
         - check
-        - test_published_artifacts
         - agent_integration_tests
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
         - test_inst_latest
-        - muzzle
         - profiling
         - debugger
         - system-tests

.gitlab-ci.yml

@@ -23,15 +23,24 @@ variables:
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
+  JAVA_BUILD_IMAGE_VERSION: "v25.01"
 
 default:
   tags: [ "arch:amd64" ]
 
+.fan_in:
+  stage: tests
+  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204-slim:release
+  script:
+    - echo "done"
+
 .gradle_build: &gradle_build
-  image: ghcr.io/datadog/dd-trace-java-docker-build:v25.01-base
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
+  stage: build
   variables:
-    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
-    GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
+    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    MAVEN_OPTS: "-Xms64M -Xmx512M"
+    GRADLE_WORKERS: 2
     KUBERNETES_CPU_REQUEST: 8
     KUBERNETES_MEMORY_REQUEST: 6Gi
   cache:
@@ -42,14 +51,15 @@ default:
         - .gradle/caches
         - .gradle/notifications
       policy: $DEPENDENCY_CACHE_POLICY
-    - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
+    - key: $CI_PIPELINE_ID-$BUILD_CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
       paths:
         - .gradle/caches/$GRADLE_VERSION
         - .gradle/$GRADLE_VERSION/executionHistory
         - workspace
       policy: $BUILD_CACHE_POLICY
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
+    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
@@ -61,21 +71,24 @@ default:
 
 build:
   extends: .gradle_build
-  stage: build
   variables:
     BUILD_CACHE_POLICY: push
+    BUILD_CACHE_TYPE: lib
     DEPENDENCY_CACHE_POLICY: pull
   script:
-    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
+    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks; fi
+    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS
     - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
     - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
     - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
   artifacts:
+    when: always
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'
       - 'workspace/dd-trace-api/build/libs/*.jar'
       - 'workspace/dd-trace-ot/build/libs/*.jar'
       - 'upstream.env'
+      - '.gradle/daemon/*/*.out.log'
     reports:
       dotenv: build.env
 
@@ -90,6 +103,94 @@ build_and_populate_dep_cache:
     - when: manual
       allow_failure: true
 
+spotless:
+  extends: .gradle_build
+  stage: tests
+  needs: []
+  script:
+    - export JAVA_HOME=$JAVA_11_HOME
+    - ./gradlew spotlessCheck $GRADLE_ARGS
+
+test_published_artifacts:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests
+  stage: tests
+  needs: [ build ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: lib
+  script:
+    - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+    - rm -rf "${mvn_local_repo}/com/datadoghq"
+    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
+    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew publishToMavenLocal $GRADLE_ARGS
+    - cd test-published-dependencies
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew check --info $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./check_reports
+
+muzzle:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  parallel: 8
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - echo CI_NODE_INDEX=$CI_NODE_INDEX
+    - echo CI_NODE_TOTAL=CI_NODE_TOTAL
+    - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
+    - sort workspace/build/muzzleTasks > sortedMuzzleTasks
+    - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
+    - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1))
+    - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+muzzle-dep-report:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_muzzle_deps.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+required:
+  extends: .fan_in
+  needs:
+    - spotless
+    - muzzle
+    - test_published_artifacts
+
 deploy_to_profiling_backend:
   stage: publish
   needs: [ build ]
@@ -165,6 +266,8 @@ deploy_to_sonatype:
   extends: .gradle_build
   stage: publish
   needs: [ build ]
+  variables:
+    BUILD_CACHE_TYPE: lib
   rules:
     - if: '$POPULATE_CACHE'
       when: never
@@ -180,7 +283,7 @@ deploy_to_sonatype:
     - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
-    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository $GRADLE_ARGS
+    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
   artifacts:
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'