diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml new file mode 100644 index 00000000..78c93442 --- /dev/null +++ b/.github/workflows/sandbox-capabilities.yml @@ -0,0 +1,258 @@ +name: Sandbox Capabilities Matrix + +on: + workflow_dispatch: + inputs: + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + iterations: + description: 'Iterations per provider' + required: false + default: '1' + concurrency: + description: 'Concurrent sandboxes per provider' + required: false + default: '1' + publish: + description: 'Commit and push results to the repo' + required: false + default: true + type: boolean + +concurrency: + group: sandbox-capabilities + cancel-in-progress: true + +permissions: {} + +jobs: + bench: + name: Capabilities ${{ matrix.provider }} + permissions: + contents: read + id-token: write + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + # runs-on: self-hosted + timeout-minutes: 30 + env: + SHOULD_RUN: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider }} + strategy: + fail-fast: false + matrix: + provider: + - archil + - arker + - beam + - blaxel + - cloud-run + - cloudflare + - codesandbox + - createos + - daytona + - declaw + - e2b + - hopx + - isorun + - lightning + - modal + - mosaic + - namespace + - northflank + - opencomputer + - runloop + - run-cloud + - sandbox0 + - sail + - sprites + - superserve + - tenki + - tensorlake + - upstash + - vercel + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + if: env.SHOULD_RUN == 'true' + with: + persist-credentials: false + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + if: env.SHOULD_RUN == 'true' + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + if: env.SHOULD_RUN == 'true' + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 + if: env.SHOULD_RUN == 'true' + - name: Install dependencies + if: env.SHOULD_RUN == 'true' + run: pnpm install --frozen-lockfile + - name: Clear stale results from checkout + if: env.SHOULD_RUN == 'true' + run: rm -rf results/ + - name: Run capabilities benchmark + if: env.SHOULD_RUN == 'true' + env: + ITERATIONS: ${{ github.event.inputs.iterations || '1' }} + CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }} + PROVIDER: ${{ matrix.provider }} + run: | + COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + case "$PROVIDER" in + archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) PROVIDER_KEYS='ARKER_API_KEY' ;; + beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) PROVIDER_KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) PROVIDER_KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) PROVIDER_KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) PROVIDER_KEYS='CSB_API_KEY' ;; + createos) PROVIDER_KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) PROVIDER_KEYS='DAYTONA_API_KEY' ;; + declaw) PROVIDER_KEYS='DECLAW_API_KEY' ;; + e2b) PROVIDER_KEYS='E2B_API_KEY' ;; + hopx) PROVIDER_KEYS='HOPX_API_KEY' ;; + isorun) PROVIDER_KEYS='ISORUN_API_KEY' ;; + lightning) PROVIDER_KEYS='LIGHTNING_API_KEY' ;; + modal) PROVIDER_KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + mosaic) PROVIDER_KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) PROVIDER_KEYS='NSC_TOKEN' ;; + northflank) PROVIDER_KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) PROVIDER_KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + runloop) PROVIDER_KEYS='RUNLOOP_API_KEY' ;; + run-cloud) PROVIDER_KEYS='RUN_CLOUD_API_KEY' ;; + sandbox0) PROVIDER_KEYS='SANDBOX0_TOKEN' ;; + sail) PROVIDER_KEYS='SAIL_API_KEY' ;; + sprites) PROVIDER_KEYS='SPRITES_TOKEN' ;; + superserve) PROVIDER_KEYS='SUPERSERVE_API_KEY' ;; + tenki) PROVIDER_KEYS='TENKI_API_KEY' ;; + tensorlake) PROVIDER_KEYS='TENSORLAKE_API_KEY' ;; + upstash) PROVIDER_KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) PROVIDER_KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) PROVIDER_KEYS='' ;; + esac + if [ -n "$PROVIDER_KEYS" ]; then + KEYS="^(${COMMON_KEYS}|${PROVIDER_KEYS})=" + else + KEYS="^(${COMMON_KEYS})=" + fi + . benchmarks/scripts/load-vault-secrets.sh "$KEYS" + + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ + --provider "$PROVIDER" \ + --iterations "$ITERATIONS" \ + --concurrency "$CONCURRENCY" \ + --run-key "$RUN_KEY" + - name: Upload results + if: always() && env.SHOULD_RUN == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: results-capabilities-${{ matrix.provider }} + path: results/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect Capabilities Results + runs-on: namespace-profile-default + # runs-on: self-hosted + needs: bench + if: always() + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Download all results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + path: artifacts/ + pattern: results-capabilities-* + - name: Merge results + continue-on-error: true + run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts + - name: Write GitHub Step Summary + if: always() + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const latestPath = path.join('results', 'sandbox-capabilities', 'latest.json'); + if (!fs.existsSync(latestPath)) { + core.summary.addHeading('Sandbox Capabilities Matrix', 2); + core.summary.addRaw('No capabilities results were generated.'); + await core.summary.write(); + return; + } + + const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + const results = data.results || []; + const featureNames = results.length > 0 ? Object.keys(results[0].features || {}) : []; + + const summary = core.summary; + summary.addHeading('Sandbox Capabilities Matrix', 2); + summary.addRaw(`

Timestamp: ${data.timestamp || 'unknown'}
Providers: ${results.length}

`); + + const header = ['Provider', ...featureNames]; + const rows = results.map(({ provider, features }) => { + return [ + provider, + ...featureNames.map((name) => (features && features[name] && features[name].passed ? '✅' : '❌')), + ]; + }); + summary.addTable([header, ...rows]); + + const failures = results + .map(({ provider, features }) => ({ + provider, + failed: Object.entries(features || {}) + .filter(([, value]) => !value.passed) + .map(([name, value]) => ({ name, error: value.error || 'failed' })), + })) + .filter((entry) => entry.failed.length > 0); + + if (failures.length > 0) { + summary.addHeading('Failure details', 3); + for (const { provider, failed } of failures) { + const content = failed + .map((failure) => `- **${failure.name}**: ${failure.error}`) + .join('\n'); + summary.addDetails(`${provider} (${failed.length} failures)`, content); + } + } + + await summary.write(); + - name: Commit and push + if: github.event.inputs.publish != 'false' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + mkdir -p results/sandbox-capabilities + git add results/sandbox-capabilities/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update sandbox capabilities results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1