CI: use EngFlow's opal cluster for remote execution

.bazelrc

@@ -3,3 +3,47 @@ common --enable_platform_specific_config
 # Needed for "@grpc//:grpc++" used by language/cc/testdata/cc_grpc_library.
 # Fixes "aligned allocation/deallocation function of type '...' is only available on macOS 10.13 or newer" errors.
 build:macos --cxxopt=-mmacosx-version-min=10.13
+
+# EngFlow common flags from https://docs.engflow.com/re/client/bazel-first-time.html#4-set-up-bazelrc
+build:engflow --define=EXECUTOR=remote
+build:engflow --disk_cache=
+build:engflow --experimental_inmemory_dotd_files
+build:engflow --experimental_inmemory_jdeps_files
+build:engflow --incompatible_strict_action_env=true
+build:engflow --remote_timeout=600
+build:engflow --nolegacy_important_outputs
+build:engflow --grpc_keepalive_time=30s
+build:engflow --experimental_remote_cache_compression=true
+build:engflow --execution_log_compact_file=execution_log.binpb.zst
+build:engflow --experimental_build_event_upload_strategy=remote
+
+# Additional clang toolchain options for automatically configured toolchains.
+build --cxxopt="-std=c++17"
+build --host_cxxopt="-std=c++17"
+build:clang --cxxopt="-fno-exceptions"
+build:clang --host_cxxopt="-fno-exceptions"
+build:clang --cxxopt="-fvisibility=hidden"
+build:clang --host_cxxopt="-fvisibility=hidden"
+
+# Platform-specific options for each supported platform.
+build:remote_linux_x64 --extra_execution_platforms=//platform/linux_x64
+build:remote_linux_x64 --host_platform=//platform/linux_x64
+build:remote_linux_x64 --platforms=//platform/linux_x64
+build:remote_linux_x64 --extra_toolchains=//platform/linux_x64:cc-toolchain
+
+build:remote_macos_arm64 --config=clang
+build:remote_macos_arm64 --host_platform=//platform/macos_arm64
+build:remote_macos_arm64 --platforms=//platform/macos_arm64
+build:remote_macos_arm64 --macos_minimum_os=14
+build:remote_macos_arm64 --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build:remote_macos_arm64 --host_action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+# EngFlow remote cluster configuration
+build:engflow --remote_instance_name=gazelle_cc
+build:engflow --remote_executor=grpcs://opal.cluster.engflow.com
+build:engflow --bes_instance_name=gazelle_cc
+build:engflow --bes_backend=grpcs://opal.cluster.engflow.com
+build:engflow --bes_results_url=https://opal.cluster.engflow.com/invocations/gazelle_cc/
+
+# Add authentication flags to this file, not checked in.
+try-import .bazelrc.user

.bazelrc.ci

@@ -0,0 +1 @@
+build:engflow --credential_helper=opal.cluster.engflow.com=%workspace%/scripts/bazel_credential_helper.sh

.github/workflows/ci.yaml

@@ -10,78 +10,173 @@ concurrency:
   group: ci-${{ github.head_ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+# Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467
+env:
+  BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }}  
+
 jobs:
-  build:
+  build_macos:
+    runs-on:
+      - "self-hosted"
+      - "os=macos"
+      - "arch=arm64"
+      - "os_distribution=sonoma"
+      - "os_version=14"
+      - "revision=c12cc4655257fcf7da86fe06e87542b7e6814a1f"
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up authentication
+        shell: bash
+        run: "cp .bazelrc.ci .bazelrc.user"
+      - name: Test
+        run: |
+          bazel test --test_output=errors --config=engflow --config=remote_macos_arm64 //...
+
+  build_linux:
+    runs-on:
+      - "self-hosted"
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_ci-runners-test-matrix_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
+    timeout-minutes: 10
     strategy:
-      # Keep in sync with configurations in .bcr/presubmit.yml. We cannot run
-      # those tests in CI: they are post-release only. So we need to cover the
-      # same configurations here.
+      fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-26]
-        # "" means use the version in .bazelversion. Add 8.x if that changes.
         bazel-version: ["", "7.x", "last_rc"]
         include:
-          # TODO #197: Turn on compilation tests again
           - bazel-version: "last_rc"
             custom-bazelrc: ".bazelrc-9"
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 30
     env:
       USE_BAZEL_VERSION: ${{ matrix.bazel-version }}
       BAZELRC: ${{ matrix.custom-bazelrc }}
     steps:
       - uses: actions/checkout@v4
-      - name: Build
-        run: |
-          bazel build //...
+      - name: Set up authentication
+        shell: bash
+        run: "cp .bazelrc.ci .bazelrc.user"
       - name: Test
         run: |
-          bazel test --test_output=errors //...
+          bazel test --test_output=errors --config=engflow --config=remote_macos_arm64 //...
 
-  # Imitate the BCR test in .bcr/presubmit.yml. Keep in sync.
-  bcr_test_module:
+  # Imitate the BCR test in .bcr/presubmit.yml.
+  # Keep in sync with configurations in .bcr/presubmit.yml. We cannot run
+  # those tests in CI: they are post-release only. So we need to cover the
+  # same configurations here.
+  bcr_test_module_macos:
+    runs-on:
+      - "self-hosted"
+      - "os=macos"
+      - "arch=arm64"
+      - "os_distribution=sonoma"
+      - "os_version=14"
+      - "revision=c12cc4655257fcf7da86fe06e87542b7e6814a1f"
+    timeout-minutes: 10
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-26]
         bazel-version: ["7.*", "8.*", "9.*"]
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 15
+    env:
+      USE_BAZEL_VERSION: "${{ matrix.bazel-version }}"
+    defaults:
+      run:
+        working-directory: "example/bzlmod"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up authentication
+        shell: bash
+        run: "cp ../../.bazelrc.ci .bazelrc.user"
+      - name: Run Gazelle
+        run: "bazel run --config=engflow --config=remote_macos_arm64 //:gazelle"
+      - name: Build
+        run: "bazel build --config=engflow --config=remote_macos_arm64 //..."
+      - name: Test
+        run: "bazel test --config=engflow --config=remote_macos_arm64 //..."
+
+  bcr_test_module_linux:
+    runs-on:
+      - "self-hosted"
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_ci-runners-test-matrix_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        bazel-version: ["7.*", "8.*", "9.*"]
+    timeout-minutes: 10
     env:
       USE_BAZEL_VERSION: ${{ matrix.bazel-version }}
     defaults:
       run:
         working-directory: "example/bzlmod"
     steps:
       - uses: actions/checkout@v4
+      - name: Set up authentication
+        shell: bash
+        run: "cp ../../.bazelrc.ci .bazelrc.user"
       - name: Run Gazelle
-        run: "bazel run //:gazelle"
+        run: "bazel run --config=engflow --config=remote_linux_x64 //:gazelle"
       - name: Build
-        run: "bazel build //..."
+        run: "bazel build --config=engflow --config=remote_linux_x64 //..."
       - name: Test
-        run: "bazel test //..."
+        run: "bazel test --config=engflow --config=remote_linux_x64 //..."
 
   test-indexers:
-    runs-on: ubuntu-24.04
+    runs-on:
+      - "self-hosted"
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_ci-runners-test-matrix_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
     timeout-minutes: 15
     steps:
     - uses: actions/checkout@v4
+    - name: Set up authentication
+      shell: bash
+      run: "cp .bazelrc.ci .bazelrc.user"
     - name: Install Conan
       run: |
         python3 -m pip install --upgrade pip
         pip install conan
     - name: Execute manual integration tests
-      run: bazel test --test_output=errors //index:integration_tests
-          
+      run: bazel test --test_output=errors --config=engflow --config=remote_linux_x64 //index:integration_tests
+
   test-e2e:
-    runs-on: ubuntu-24.04
+    runs-on:
+      - "self-hosted"
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_ci-runners-test-matrix_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
     timeout-minutes: 15
     steps:
     - uses: actions/checkout@v4  
+    - name: Set up authentication
+      shell: bash
+      run: "cp .bazelrc.ci .bazelrc.user"
     - name: Execute end to end tests
       run: bash .github/workflows/test_e2e.sh
 
   license-headers:
-    runs-on: ubuntu-24.04
+    runs-on:
+      - "self-hosted"
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_ci-runners-test-matrix_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4

.github/workflows/test_e2e.sh

@@ -5,14 +5,19 @@ set -o errexit -o nounset -o pipefail
 scriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 rootDir=$(realpath "$scriptDir/../..")
 
+BAZEL_CONFIG_FLAGS=(
+  --config=engflow
+  --config=remote_linux_x64
+)
+
 function runBazelCommandTreatingWarningsAsErrors() {
   local OUTPUT_BASE_DIR=$(bazel info output_base)
   local BAZEL_COMMAND_LOG_FILE="$OUTPUT_BASE_DIR/command.log"
   local BAZEL_CMD="$1"
   shift
 
   # Disable colors and control characters to make grep work properly
-  command bazel "$BAZEL_CMD" --color=no --curses=no "$@"
+  command bazel "$BAZEL_CMD" "${BAZEL_CONFIG_FLAGS[@]}" "$@"
 
   # Expect no warnings in the command output
   local COLLECTED_WARNINGS=$(grep "^WARNING" "$BAZEL_COMMAND_LOG_FILE" || true)

.gitignore

@@ -5,6 +5,7 @@ sandbox/*
 .scala-build
 .metals
 .bsp
+.bazelrc.user
 
 # macOS
 .DS_Store

example/bzlmod/.bazelrc

@@ -0,0 +1,49 @@
+common --enable_platform_specific_config
+
+# Needed for "@grpc//:grpc++" used by language/cc/testdata/cc_grpc_library.
+# Fixes "aligned allocation/deallocation function of type '...' is only available on macOS 10.13 or newer" errors.
+build:macos --cxxopt=-mmacosx-version-min=10.13
+
+# EngFlow common flags from https://docs.engflow.com/re/client/bazel-first-time.html#4-set-up-bazelrc
+build:engflow --define=EXECUTOR=remote
+build:engflow --disk_cache=
+build:engflow --experimental_inmemory_dotd_files
+build:engflow --experimental_inmemory_jdeps_files
+build:engflow --incompatible_strict_action_env=true
+build:engflow --remote_timeout=600
+build:engflow --nolegacy_important_outputs
+build:engflow --grpc_keepalive_time=30s
+build:engflow --experimental_remote_cache_compression=true
+build:engflow --execution_log_compact_file=execution_log.binpb.zst
+build:engflow --experimental_build_event_upload_strategy=remote
+
+# Additional clang toolchain options for automatically configured toolchains.
+build --cxxopt="-std=c++17"
+build --host_cxxopt="-std=c++17"
+build:clang --cxxopt="-fno-exceptions"
+build:clang --host_cxxopt="-fno-exceptions"
+build:clang --cxxopt="-fvisibility=hidden"
+build:clang --host_cxxopt="-fvisibility=hidden"
+
+# Platform-specific options for each supported platform.
+build:remote_linux_x64 --extra_execution_platforms=@gazelle_cc//linux_x64
+build:remote_linux_x64 --host_platform=@gazelle_cc//linux_x64
+build:remote_linux_x64 --platforms=@gazelle_cc//linux_x64
+build:remote_linux_x64 --extra_toolchains=@gazelle_cc//linux_x64:cc-toolchain
+
+build:remote_macos_arm64 --config=clang
+build:remote_macos_arm64 --host_platform=@gazelle_cc//macos_arm64
+build:remote_macos_arm64 --platforms=@gazelle_cc//macos_arm64
+build:remote_macos_arm64 --macos_minimum_os=14
+build:remote_macos_arm64 --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build:remote_macos_arm64 --host_action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+# EngFlow remote cluster configuration
+build:engflow --remote_instance_name=gazelle_cc
+build:engflow --remote_executor=grpcs://opal.cluster.engflow.com
+build:engflow --bes_instance_name=gazelle_cc
+build:engflow --bes_backend=grpcs://opal.cluster.engflow.com
+build:engflow --bes_results_url=https://opal.cluster.engflow.com/invocations/gazelle_cc/
+
+# Add authentication flags to this file, not checked in.
+try-import .bazelrc.user