ci: add sandbox-capabilities workflow for GitHub Actions smoke test

.github/workflows/sandbox-capabilities.yml

@@ -0,0 +1,258 @@
+name: Sandbox Capabilities Matrix
+
+on:
+  workflow_dispatch:
+    inputs:
+      provider:
+        description: 'Provider to run (leave empty for all)'
+        required: false
+        default: ''
+      iterations:
+        description: 'Iterations per provider'
+        required: false
+        default: '1'
+      concurrency:
+        description: 'Concurrent sandboxes per provider'
+        required: false
+        default: '1'
+      publish:
+        description: 'Commit and push results to the repo'
+        required: false
+        default: true
+        type: boolean
+
+concurrency:
+  group: sandbox-capabilities
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  bench:
+    name: Capabilities ${{ matrix.provider }}
+    permissions:
+      contents: read
+      id-token: write
+    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
+    # runs-on: self-hosted
+    timeout-minutes: 30
+    env:
+      SHOULD_RUN: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider }}
+    strategy:
+      fail-fast: false
+      matrix:
+        provider:
+          - archil
+          - arker
+          - beam
+          - blaxel
+          - cloud-run
+          - cloudflare
+          - codesandbox
+          - createos
+          - daytona
+          - declaw
+          - e2b
+          - hopx
+          - isorun
+          - lightning
+          - modal
+          - mosaic
+          - namespace
+          - northflank
+          - opencomputer
+          - runloop
+          - run-cloud
+          - sandbox0
+          - sail
+          - sprites
+          - superserve
+          - tenki
+          - tensorlake
+          - upstash
+          - vercel
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        if: env.SHOULD_RUN == 'true'
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
+        if: env.SHOULD_RUN == 'true'
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        if: env.SHOULD_RUN == 'true'
+        with:
+          node-version: 24
+          cache: 'pnpm'
+      - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12
+        if: env.SHOULD_RUN == 'true'
+      - name: Install dependencies
+        if: env.SHOULD_RUN == 'true'
+        run: pnpm install --frozen-lockfile
+      - name: Clear stale results from checkout
+        if: env.SHOULD_RUN == 'true'
+        run: rm -rf results/
+      - name: Run capabilities benchmark
+        if: env.SHOULD_RUN == 'true'
+        env:
+          ITERATIONS: ${{ github.event.inputs.iterations || '1' }}
+          CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }}
+          PROVIDER: ${{ matrix.provider }}
+        run: |
+          COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY'
+          case "$PROVIDER" in
+            archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;;
+            arker) PROVIDER_KEYS='ARKER_API_KEY' ;;
+            beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;;
+            blaxel) PROVIDER_KEYS='BL_API_KEY|BL_WORKSPACE' ;;
+            cloud-run) PROVIDER_KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;;
+            cloudflare) PROVIDER_KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;;
+            codesandbox) PROVIDER_KEYS='CSB_API_KEY' ;;
+            createos) PROVIDER_KEYS='CREATEOS_SANDBOX_API_KEY' ;;
+            daytona) PROVIDER_KEYS='DAYTONA_API_KEY' ;;
+            declaw) PROVIDER_KEYS='DECLAW_API_KEY' ;;
+            e2b) PROVIDER_KEYS='E2B_API_KEY' ;;
+            hopx) PROVIDER_KEYS='HOPX_API_KEY' ;;
+            isorun) PROVIDER_KEYS='ISORUN_API_KEY' ;;
+            lightning) PROVIDER_KEYS='LIGHTNING_API_KEY' ;;
+            modal) PROVIDER_KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;;
+            mosaic) PROVIDER_KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;;
+            namespace) PROVIDER_KEYS='NSC_TOKEN' ;;
+            northflank) PROVIDER_KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;;
+            opencomputer) PROVIDER_KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;;
+            runloop) PROVIDER_KEYS='RUNLOOP_API_KEY' ;;
+            run-cloud) PROVIDER_KEYS='RUN_CLOUD_API_KEY' ;;
+            sandbox0) PROVIDER_KEYS='SANDBOX0_TOKEN' ;;
+            sail) PROVIDER_KEYS='SAIL_API_KEY' ;;
+            sprites) PROVIDER_KEYS='SPRITES_TOKEN' ;;
+            superserve) PROVIDER_KEYS='SUPERSERVE_API_KEY' ;;
+            tenki) PROVIDER_KEYS='TENKI_API_KEY' ;;
+            tensorlake) PROVIDER_KEYS='TENSORLAKE_API_KEY' ;;
+            upstash) PROVIDER_KEYS='UPSTASH_BOX_API_KEY' ;;
+            vercel) PROVIDER_KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;;
+            *) PROVIDER_KEYS='' ;;
+          esac
+          if [ -n "$PROVIDER_KEYS" ]; then
+            KEYS="^(${COMMON_KEYS}|${PROVIDER_KEYS})="
+          else
+            KEYS="^(${COMMON_KEYS})="
+          fi
+          . benchmarks/scripts/load-vault-secrets.sh "$KEYS"
+
+          RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \
+            --provider "$PROVIDER" \
+            --iterations "$ITERATIONS" \
+            --concurrency "$CONCURRENCY" \
+            --run-key "$RUN_KEY"
+      - name: Upload results
+        if: always() && env.SHOULD_RUN == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: results-capabilities-${{ matrix.provider }}
+          path: results/
+          if-no-files-found: ignore
+          retention-days: 7
+
+  collect:
+    name: Collect Capabilities Results
+    runs-on: namespace-profile-default
+    # runs-on: self-hosted
+    needs: bench
+    if: always()
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: 24
+          cache: 'pnpm'
+      - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Download all results
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          path: artifacts/
+          pattern: results-capabilities-*
+      - name: Merge results
+        continue-on-error: true
+        run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts
+      - name: Write GitHub Step Summary
+        if: always()
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const latestPath = path.join('results', 'sandbox-capabilities', 'latest.json');
+            if (!fs.existsSync(latestPath)) {
+              core.summary.addHeading('Sandbox Capabilities Matrix', 2);
+              core.summary.addRaw('No capabilities results were generated.');
+              await core.summary.write();
+              return;
+            }
+
+            const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8'));
+            const results = data.results || [];
+            const featureNames = results.length > 0 ? Object.keys(results[0].features || {}) : [];
+
+            const summary = core.summary;
+            summary.addHeading('Sandbox Capabilities Matrix', 2);
+            summary.addRaw(`<p>Timestamp: ${data.timestamp || 'unknown'}<br>Providers: ${results.length}</p>`);
+
+            const header = ['Provider', ...featureNames];
+            const rows = results.map(({ provider, features }) => {
+              return [
+                provider,
+                ...featureNames.map((name) => (features && features[name] && features[name].passed ? '✅' : '❌')),
+              ];
+            });
+            summary.addTable([header, ...rows]);
+
+            const failures = results
+              .map(({ provider, features }) => ({
+                provider,
+                failed: Object.entries(features || {})
+                  .filter(([, value]) => !value.passed)
+                  .map(([name, value]) => ({ name, error: value.error || 'failed' })),
+              }))
+              .filter((entry) => entry.failed.length > 0);
+
+            if (failures.length > 0) {
+              summary.addHeading('Failure details', 3);
+              for (const { provider, failed } of failures) {
+                const content = failed
+                  .map((failure) => `- **${failure.name}**: ${failure.error}`)
+                  .join('\n');
+                summary.addDetails(`${provider} (${failed.length} failures)`, content);
+              }
+            }
+
+            await summary.write();
+      - name: Commit and push
+        if: github.event.inputs.publish != 'false'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          mkdir -p results/sandbox-capabilities
+          git add results/sandbox-capabilities/
+          git diff --cached --quiet && echo "No changes to commit" && exit 0
+          git commit -m "chore: update sandbox capabilities results [skip ci]"
+
+          branch="${GITHUB_REF#refs/heads/}"
+          for attempt in 1 2 3 4 5; do
+            git fetch origin "${branch}"
+            git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; }
+            if git push origin "HEAD:${branch}"; then
+              echo "Pushed on attempt ${attempt}"
+              exit 0
+            fi
+            echo "Push rejected (attempt ${attempt}); will rebase and retry"
+          done
+          echo "Failed to push after multiple attempts" >&2
+          exit 1