From 3807e19494b84a90e9fa98868e48cb87e17aaeed Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:10:13 +0000 Subject: [PATCH 1/6] Add sandbox capabilities workflow for GitHub Actions smoke test Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 251 +++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 .github/workflows/sandbox-capabilities.yml diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml new file mode 100644 index 00000000..f5a36bcd --- /dev/null +++ b/.github/workflows/sandbox-capabilities.yml @@ -0,0 +1,251 @@ +name: Sandbox Capabilities Matrix + +on: + schedule: + - cron: '0 10 * * 1' # Weekly on Monday at 10am UTC / 5am CDT + workflow_dispatch: + inputs: + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + iterations: + description: 'Iterations per provider' + required: false + default: '1' + concurrency: + description: 'Concurrent sandboxes per provider' + required: false + default: '1' + publish: + description: 'Commit and push results to the repo' + required: false + default: true + type: boolean + +concurrency: + group: sandbox-capabilities + cancel-in-progress: true + +permissions: {} + +jobs: + bench: + name: Capabilities ${{ matrix.provider }} + permissions: + contents: read + id-token: write + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + # runs-on: self-hosted + timeout-minutes: 30 + env: + SHOULD_RUN: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider }} + strategy: + fail-fast: false + matrix: + provider: + - archil + - arker + - beam + - blaxel + - cloud-run + - cloudflare + - codesandbox + - createos + - daytona + - declaw + - e2b + - hopx + - isorun + - lightning + - modal + - mosaic + - namespace + - northflank + - opencomputer + - runloop + - run-cloud + - sandbox0 + - sail + - sprites + - superserve + - tenki + - tensorlake + - upstash + - vercel + steps: + - uses: actions/checkout@v4 + if: env.SHOULD_RUN == 'true' + - uses: pnpm/action-setup@v4 + if: env.SHOULD_RUN == 'true' + - uses: actions/setup-node@v4 + if: env.SHOULD_RUN == 'true' + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@v0 + if: env.SHOULD_RUN == 'true' + - name: Install dependencies + if: env.SHOULD_RUN == 'true' + run: pnpm install --frozen-lockfile + - name: Clear stale results from checkout + if: env.SHOULD_RUN == 'true' + run: rm -rf results/ + - name: Run capabilities benchmark + if: env.SHOULD_RUN == 'true' + run: | + case '${{ matrix.provider }}' in + archil) KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) KEYS='ARKER_API_KEY' ;; + beam) KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) KEYS='CSB_API_KEY' ;; + createos) KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) KEYS='DAYTONA_API_KEY' ;; + declaw) KEYS='DECLAW_API_KEY' ;; + e2b) KEYS='E2B_API_KEY' ;; + hopx) KEYS='HOPX_API_KEY' ;; + isorun) KEYS='ISORUN_API_KEY' ;; + lightning) KEYS='LIGHTNING_API_KEY' ;; + modal) KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + mosaic) KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) KEYS='NSC_TOKEN' ;; + northflank) KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + runloop) KEYS='RUNLOOP_API_KEY' ;; + run-cloud) KEYS='RUN_CLOUD_API_KEY' ;; + sandbox0) KEYS='SANDBOX0_TOKEN' ;; + sail) KEYS='SAIL_API_KEY' ;; + sprites) KEYS='SPRITES_TOKEN' ;; + superserve) KEYS='SUPERSERVE_API_KEY' ;; + tenki) KEYS='TENKI_API_KEY' ;; + tensorlake) KEYS='TENSORLAKE_API_KEY' ;; + upstash) KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) KEYS='' ;; + esac + if [ -n "$KEYS" ]; then + KEYS="BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY|${KEYS}" + else + KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + fi + . benchmarks/scripts/load-vault-secrets.sh "$KEYS" + + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ + --provider '${{ matrix.provider }}' \ + --iterations '${{ github.event.inputs.iterations || '1' }}' \ + --concurrency '${{ github.event.inputs.concurrency || '1' }}' \ + --run-key "$RUN_KEY" + - name: Upload results + if: always() && env.SHOULD_RUN == 'true' + uses: actions/upload-artifact@v4 + with: + name: results-capabilities-${{ matrix.provider }} + path: results/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect Capabilities Results + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + # runs-on: self-hosted + needs: bench + if: always() + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@v0 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Download all results + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: results-capabilities-* + - name: Merge results + run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts + - name: Write GitHub Step Summary + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const latestPath = path.join('results', 'sandbox-capabilities', 'latest.json'); + if (!fs.existsSync(latestPath)) { + core.summary.addHeading('Sandbox Capabilities Matrix', 2); + core.summary.addRaw('No capabilities results were generated.'); + await core.summary.write(); + return; + } + + const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + const results = data.results || []; + const featureNames = results.length > 0 ? Object.keys(results[0].features || {}) : []; + + const summary = core.summary; + summary.addHeading('Sandbox Capabilities Matrix', 2); + summary.addRaw(`

Timestamp: ${data.timestamp || 'unknown'}
Providers: ${results.length}

`); + + const header = ['Provider', ...featureNames]; + const rows = results.map(({ provider, features }) => { + return [ + provider, + ...featureNames.map((name) => (features && features[name] && features[name].passed ? '✅' : '❌')), + ]; + }); + summary.addTable([header, ...rows]); + + const failures = results + .map(({ provider, features }) => ({ + provider, + failed: Object.entries(features || {}) + .filter(([, value]) => !value.passed) + .map(([name, value]) => ({ name, error: value.error || 'failed' })), + })) + .filter((entry) => entry.failed.length > 0); + + if (failures.length > 0) { + summary.addHeading('Failure details', 3); + for (const { provider, failed } of failures) { + const content = failed + .map((failure) => `- **${failure.name}**: ${failure.error}`) + .join('\n'); + summary.addDetails(`${provider} (${failed.length} failures)`, content); + } + } + + await summary.write(); + - name: Commit and push + if: github.event.inputs.publish != 'false' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add results/sandbox-capabilities/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update sandbox capabilities results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1 From 835285a2d89f7417b4f8e62f26906b9473827f39 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:24:27 +0000 Subject: [PATCH 2/6] ci: harden sandbox-capabilities workflow for smoke test Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 96 +++++++++++----------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index f5a36bcd..5d1a32c7 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -1,8 +1,6 @@ name: Sandbox Capabilities Matrix on: - schedule: - - cron: '0 10 * * 1' # Weekly on Monday at 10am UTC / 5am CDT workflow_dispatch: inputs: provider: @@ -74,16 +72,18 @@ jobs: - upstash - vercel steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: env.SHOULD_RUN == 'true' - - uses: pnpm/action-setup@v4 + with: + persist-credentials: false + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 if: env.SHOULD_RUN == 'true' - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' with: node-version: 24 cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 if: env.SHOULD_RUN == 'true' - name: Install dependencies if: env.SHOULD_RUN == 'true' @@ -94,42 +94,43 @@ jobs: - name: Run capabilities benchmark if: env.SHOULD_RUN == 'true' run: | + COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' case '${{ matrix.provider }}' in - archil) KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; - arker) KEYS='ARKER_API_KEY' ;; - beam) KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; - blaxel) KEYS='BL_API_KEY|BL_WORKSPACE' ;; - cloud-run) KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; - cloudflare) KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; - codesandbox) KEYS='CSB_API_KEY' ;; - createos) KEYS='CREATEOS_SANDBOX_API_KEY' ;; - daytona) KEYS='DAYTONA_API_KEY' ;; - declaw) KEYS='DECLAW_API_KEY' ;; - e2b) KEYS='E2B_API_KEY' ;; - hopx) KEYS='HOPX_API_KEY' ;; - isorun) KEYS='ISORUN_API_KEY' ;; - lightning) KEYS='LIGHTNING_API_KEY' ;; - modal) KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; - mosaic) KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; - namespace) KEYS='NSC_TOKEN' ;; - northflank) KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; - opencomputer) KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; - runloop) KEYS='RUNLOOP_API_KEY' ;; - run-cloud) KEYS='RUN_CLOUD_API_KEY' ;; - sandbox0) KEYS='SANDBOX0_TOKEN' ;; - sail) KEYS='SAIL_API_KEY' ;; - sprites) KEYS='SPRITES_TOKEN' ;; - superserve) KEYS='SUPERSERVE_API_KEY' ;; - tenki) KEYS='TENKI_API_KEY' ;; - tensorlake) KEYS='TENSORLAKE_API_KEY' ;; - upstash) KEYS='UPSTASH_BOX_API_KEY' ;; - vercel) KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; - *) KEYS='' ;; + archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) PROVIDER_KEYS='ARKER_API_KEY' ;; + beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) PROVIDER_KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) PROVIDER_KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) PROVIDER_KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) PROVIDER_KEYS='CSB_API_KEY' ;; + createos) PROVIDER_KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) PROVIDER_KEYS='DAYTONA_API_KEY' ;; + declaw) PROVIDER_KEYS='DECLAW_API_KEY' ;; + e2b) PROVIDER_KEYS='E2B_API_KEY' ;; + hopx) PROVIDER_KEYS='HOPX_API_KEY' ;; + isorun) PROVIDER_KEYS='ISORUN_API_KEY' ;; + lightning) PROVIDER_KEYS='LIGHTNING_API_KEY' ;; + modal) PROVIDER_KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + mosaic) PROVIDER_KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) PROVIDER_KEYS='NSC_TOKEN' ;; + northflank) PROVIDER_KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) PROVIDER_KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + runloop) PROVIDER_KEYS='RUNLOOP_API_KEY' ;; + run-cloud) PROVIDER_KEYS='RUN_CLOUD_API_KEY' ;; + sandbox0) PROVIDER_KEYS='SANDBOX0_TOKEN' ;; + sail) PROVIDER_KEYS='SAIL_API_KEY' ;; + sprites) PROVIDER_KEYS='SPRITES_TOKEN' ;; + superserve) PROVIDER_KEYS='SUPERSERVE_API_KEY' ;; + tenki) PROVIDER_KEYS='TENKI_API_KEY' ;; + tensorlake) PROVIDER_KEYS='TENSORLAKE_API_KEY' ;; + upstash) PROVIDER_KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) PROVIDER_KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) PROVIDER_KEYS='' ;; esac - if [ -n "$KEYS" ]; then - KEYS="BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY|${KEYS}" + if [ -n "$PROVIDER_KEYS" ]; then + KEYS="^(${COMMON_KEYS}|${PROVIDER_KEYS})=" else - KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + KEYS="^(${COMMON_KEYS})=" fi . benchmarks/scripts/load-vault-secrets.sh "$KEYS" @@ -141,7 +142,7 @@ jobs: --run-key "$RUN_KEY" - name: Upload results if: always() && env.SHOULD_RUN == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: results-capabilities-${{ matrix.provider }} path: results/ @@ -158,26 +159,28 @@ jobs: contents: write id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 - name: Install dependencies run: pnpm install --frozen-lockfile - name: Download all results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts/ pattern: results-capabilities-* - name: Merge results + continue-on-error: true run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts - name: Write GitHub Step Summary - uses: actions/github-script@v7 + if: always() + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const fs = require('fs'); @@ -233,6 +236,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + mkdir -p results/sandbox-capabilities git add results/sandbox-capabilities/ git diff --cached --quiet && echo "No changes to commit" && exit 0 git commit -m "chore: update sandbox capabilities results [skip ci]" From 05901886ace93128b43e558632077929670403d1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:31:57 +0000 Subject: [PATCH 3/6] ci: pin pnpm/action-setup to v4.4.0 SHA Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 5d1a32c7..629c6dd3 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -76,7 +76,7 @@ jobs: if: env.SHOULD_RUN == 'true' with: persist-credentials: false - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 if: env.SHOULD_RUN == 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' @@ -162,7 +162,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 From ae9176942de5f3a007cefc5936a05ea61ff5be9a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:36:59 +0000 Subject: [PATCH 4/6] ci: revert pnpm/action-setup pin to v4 tag SHA Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 629c6dd3..5d1a32c7 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -76,7 +76,7 @@ jobs: if: env.SHOULD_RUN == 'true' with: persist-credentials: false - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 if: env.SHOULD_RUN == 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' @@ -162,7 +162,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 From 0c3cb0ec469c61dd7fb98c65a93ec689a5e7fe48 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:42:56 +0000 Subject: [PATCH 5/6] ci: avoid template injection by passing inputs via env vars Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 5d1a32c7..e787b873 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -93,9 +93,13 @@ jobs: run: rm -rf results/ - name: Run capabilities benchmark if: env.SHOULD_RUN == 'true' + env: + ITERATIONS: ${{ github.event.inputs.iterations || '1' }} + CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }} + PROVIDER: ${{ matrix.provider }} run: | COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' - case '${{ matrix.provider }}' in + case "$PROVIDER" in archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; arker) PROVIDER_KEYS='ARKER_API_KEY' ;; beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; @@ -136,9 +140,9 @@ jobs: RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ - --provider '${{ matrix.provider }}' \ - --iterations '${{ github.event.inputs.iterations || '1' }}' \ - --concurrency '${{ github.event.inputs.concurrency || '1' }}' \ + --provider "$PROVIDER" \ + --iterations "$ITERATIONS" \ + --concurrency "$CONCURRENCY" \ --run-key "$RUN_KEY" - name: Upload results if: always() && env.SHOULD_RUN == 'true' From e84bbae4d1af3d467def4ecf894ce6482b778f6e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:48:26 +0000 Subject: [PATCH 6/6] ci: remove unneeded vault and id-token permissions from collect job Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index e787b873..78c93442 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -155,13 +155,12 @@ jobs: collect: name: Collect Capabilities Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + runs-on: namespace-profile-default # runs-on: self-hosted needs: bench if: always() permissions: contents: write - id-token: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: