From 732e8a262308ff3f062d6bd5b9f94e0ca71e486e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:17:26 +0000 Subject: [PATCH 01/10] Add sandbox capabilities matrix benchmark Co-Authored-By: Noah Kiser --- benchmarks/sandbox/capabilities-results.ts | 68 ++++ benchmarks/sandbox/capabilities.bench.ts | 357 +++++++++++++++++++++ benchmarks/sandbox/types.ts | 16 + package.json | 1 + 4 files changed, 442 insertions(+) create mode 100644 benchmarks/sandbox/capabilities-results.ts create mode 100644 benchmarks/sandbox/capabilities.bench.ts diff --git a/benchmarks/sandbox/capabilities-results.ts b/benchmarks/sandbox/capabilities-results.ts new file mode 100644 index 00000000..944eafac --- /dev/null +++ b/benchmarks/sandbox/capabilities-results.ts @@ -0,0 +1,68 @@ +import { mkdirSync, writeFileSync, copyFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { ParticipantRecords, ResolvedRunConfig } from '@benchsdk/runner'; +import type { TaskResultRecord } from '@benchsdk/client'; +import type { ProviderCapabilityMatrix, CapabilityMatrixResult } from './types.js'; + +const RESULTS_VERSION = '1.0'; + +/** + * Prefer the task-level `features` matrix the capability task returns in + * `record.data`. As a fallback, reconstruct from per-step measurements when a + * step failed and the matrix wasn't emitted. + */ +function extractMatrix(record: TaskResultRecord): ProviderCapabilityMatrix { + const data = record.data ?? {}; + if (data && typeof data === 'object' && 'features' in data && data.features && typeof data.features === 'object' && !Array.isArray(data.features)) { + return data.features as unknown as ProviderCapabilityMatrix; + } + + const matrix: ProviderCapabilityMatrix = {}; + for (const step of record.steps ?? []) { + const stepData = step.data ?? {}; + if (stepData && typeof stepData === 'object' && 'passed' in stepData) { + const { passed, error } = stepData as { passed: boolean; error?: string }; + matrix[step.name] = passed ? { passed: true } : { passed: false, ...(error ? { error } : {}) }; + } + } + return matrix; +} + +/** Aggregate per-provider capability matrices and write dated + latest JSON. */ +export async function writeSandboxCapabilitiesResults( + participants: ParticipantRecords[], + opts: { resultsDir: string; runConfig: ResolvedRunConfig }, +): Promise { + const results: CapabilityMatrixResult[] = participants.map((participant) => { + const record = participant.records[0]; + const features = record ? extractMatrix(record) : {}; + return { provider: participant.participant, features }; + }); + + mkdirSync(opts.resultsDir, { recursive: true }); + + const timestamp = new Date().toISOString(); + const outPath = path.join(opts.resultsDir, `${timestamp.slice(0, 10)}.json`); + const latestPath = path.join(opts.resultsDir, 'latest.json'); + + const output = { + version: RESULTS_VERSION, + timestamp, + environment: { + node: process.version, + platform: os.platform(), + arch: os.arch(), + }, + config: { + benchmarkSlug: 'sandbox-capabilities', + iterations: opts.runConfig.iterations, + concurrency: opts.runConfig.concurrency, + }, + results, + }; + + writeFileSync(outPath, JSON.stringify(output, null, 2)); + copyFileSync(outPath, latestPath); + console.log(`Capabilities results written: ${outPath} -> ${latestPath}`); +} diff --git a/benchmarks/sandbox/capabilities.bench.ts b/benchmarks/sandbox/capabilities.bench.ts new file mode 100644 index 00000000..2a22e6a2 --- /dev/null +++ b/benchmarks/sandbox/capabilities.bench.ts @@ -0,0 +1,357 @@ +/** + * Sandbox capabilities matrix benchmark. + * + * Creates one sandbox per provider and probes the full ComputeSDK surface, + * recording a pass/fail (plus error text) for each named capability. Each + * probe is its own `ctx.step` with an individual try/catch so a failure in one + * capability does not abort the remaining probes. Results are aggregated into + * a feature matrix JSON under `results/sandbox-capabilities/`. + * + * bench run benchmarks/sandbox/capabilities.bench.ts + * bench run benchmarks/sandbox/capabilities.bench.ts --provider e2b,modal + */ +import '../src/env.js'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { TaskResult } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/client'; +import type { + SandboxInterface, + CommandResult, + SandboxInfo, + FileEntry, + RunCommandOptions, + CreateSandboxOptions, +} from 'computesdk'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; +import { providers } from './providers.js'; +import type { ProviderConfig, ProviderCapabilityMatrix } from './types.js'; +import { writeSandboxCapabilitiesResults } from './capabilities-results.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const CREATE_TIMEOUT_MS = 120_000; +const COMMAND_TIMEOUT_MS = 30_000; +const DESTROY_TIMEOUT_MS = 15_000; +const LIST_TIMEOUT_MS = 30_000; +const GETURL_TIMEOUT_MS = 30_000; +const SNAPSHOT_TIMEOUT_MS = 120_000; +const TEMPLATE_TIMEOUT_MS = 120_000; +const STREAM_TIMEOUT_MS = 30_000; +const FS_TIMEOUT_MS = 30_000; +const GETURL_PORT = 3000; + +interface CapabilityCompute { + readonly name: string; + readonly sandbox: { + create(options?: CreateSandboxOptions): Promise; + getById(sandboxId: string): Promise; + list(): Promise; + destroy(sandboxId: string): Promise; + }; + readonly snapshot?: { + create(sandboxId: string, options?: { name?: string; metadata?: Record }): Promise; + list(options?: { sandboxId?: string; limit?: number }): Promise; + delete(snapshotId: string): Promise; + }; + readonly template?: { + create(options: { name: string; description?: string; metadata?: Record }): Promise; + list(options?: { limit?: number }): Promise; + delete(templateId: string): Promise; + }; +} + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'sandbox-capabilities', + benchmarkName: 'Sandbox Capabilities', + iterations: 1, + concurrency: 1, + participants: providers, + onComplete: (outcome) => + writeSandboxCapabilitiesResults(outcome.participants, { + resultsDir: path.resolve(__dirname, '../../results/sandbox-capabilities'), + runConfig: outcome.config, + }), +}); + +export const task = defineTask(async (ctx): Promise => { + const { participant, step, measure } = ctx; + const compute = participant.createCompute() as CapabilityCompute; + + const rawOptions = participant.sandboxOptions ?? {}; + const getUrlPort = Array.isArray(rawOptions.ports) && rawOptions.ports.length > 0 + ? (rawOptions.ports as number[])[0] + : GETURL_PORT; + const sandboxOptions: CreateSandboxOptions = { ...rawOptions }; + + const filePath = '/tmp/capabilities-file.txt'; + const dirPath = '/tmp/capabilities-dir'; + const testContent = 'Hello, ComputeSDK filesystem!'; + + const matrix: ProviderCapabilityMatrix = {}; + let sandbox: SandboxInterface | null = null; + let createdSnapshotId: string | undefined; + let createdTemplateId: string | undefined; + + function recordFeature(name: string, passed: boolean, error?: string) { + matrix[name] = passed ? { passed: true } : { passed: false, ...(error ? { error } : {}) }; + } + + async function probe( + name: string, + fn: () => Promise, + opts: { needsSandbox?: boolean } = {}, + ): Promise { + await step(name, async () => { + if (opts.needsSandbox && !sandbox) { + const error = 'sandbox creation failed'; + measure({ passed: false, error }); + recordFeature(name, false, error); + return; + } + try { + await fn(); + measure({ passed: true }); + recordFeature(name, true); + } catch (err) { + const error = formatError(err); + measure({ passed: false, error }); + recordFeature(name, false, error); + } + }); + } + + await probe('create', async () => { + sandbox = await withTimeout( + compute.sandbox.create(sandboxOptions), + participant.timeout ?? CREATE_TIMEOUT_MS, + 'Sandbox creation timed out', + ); + }); + + try { + await probe('runCommand', async () => { + const result = await withTimeout( + sandbox!.runCommand('echo "capability-test"'), + COMMAND_TIMEOUT_MS, + 'Command timed out', + ) as CommandResult; + if (result.exitCode !== 0) { + throw new Error(`Command failed with exit code ${result.exitCode}: ${result.stderr || 'unknown'}`); + } + }, { needsSandbox: true }); + + await probe('getInfo', async () => { + const info = (await withTimeout( + sandbox!.getInfo(), + COMMAND_TIMEOUT_MS, + 'getInfo timed out', + )) as SandboxInfo; + if (!info || typeof info.id !== 'string') { + throw new Error('getInfo returned invalid sandbox info'); + } + }, { needsSandbox: true }); + + await probe('getById', async () => { + const retrieved = await withTimeout( + compute.sandbox.getById(sandbox!.sandboxId), + LIST_TIMEOUT_MS, + 'getById timed out', + ); + if (!retrieved) throw new Error('getById returned null'); + if (retrieved.sandboxId !== sandbox!.sandboxId) throw new Error('getById returned a different sandbox'); + }, { needsSandbox: true }); + + await probe('list', async () => { + const list = await withTimeout(compute.sandbox.list(), LIST_TIMEOUT_MS, 'list timed out'); + if (!Array.isArray(list)) throw new Error('list did not return an array'); + }); + + await probe('getUrl', async () => { + const url = await withTimeout( + sandbox!.getUrl({ port: getUrlPort }), + GETURL_TIMEOUT_MS, + 'getUrl timed out', + ); + if (!url || typeof url !== 'string' || !/^(https?|wss?):\/\/.+/.test(url)) { + throw new Error(`getUrl returned invalid URL: ${url}`); + } + }, { needsSandbox: true }); + + await probe('background', async () => { + const result = await withTimeout( + sandbox!.runCommand('sleep 1', { background: true } as RunCommandOptions), + COMMAND_TIMEOUT_MS, + 'Background command timed out', + ) as CommandResult; + if (result.exitCode !== 0) { + throw new Error(`Background command failed with exit code ${result.exitCode}: ${result.stderr || 'unknown'}`); + } + }, { needsSandbox: true }); + + await probe('streaming', async () => { + const chunks: string[] = []; + let firstChunkAt: number | undefined; + const startedAt = Date.now(); + const result = await withTimeout( + sandbox!.runCommand( + `sh -c 'for i in 1 2 3 4 5; do echo tick $i; sleep 1; done'`, + { onStdout: (text: string) => { firstChunkAt ??= Date.now(); chunks.push(text); } } as RunCommandOptions, + ), + STREAM_TIMEOUT_MS, + 'Streaming command timed out', + ) as CommandResult; + if (result.exitCode !== 0) { + throw new Error(`Streaming command failed with exit code ${result.exitCode}: ${result.stderr || 'unknown'}`); + } + if (firstChunkAt === undefined) throw new Error('No stdout chunk received before command completed'); + if (result.durationMs > 0 && firstChunkAt - startedAt >= result.durationMs / 2) { + throw new Error('First stdout chunk arrived too late (likely buffered until completion)'); + } + }, { needsSandbox: true }); + + await probe('writeFile', async () => { + await withTimeout( + sandbox!.filesystem.writeFile(filePath, testContent), + FS_TIMEOUT_MS, + 'writeFile timed out', + ); + }, { needsSandbox: true }); + + await probe('readFile', async () => { + const content = await withTimeout( + sandbox!.filesystem.readFile(filePath), + FS_TIMEOUT_MS, + 'readFile timed out', + ); + if (content !== testContent) { + throw new Error('readFile returned unexpected content'); + } + }, { needsSandbox: true }); + + await probe('mkdir', async () => { + await withTimeout( + sandbox!.filesystem.mkdir(dirPath), + FS_TIMEOUT_MS, + 'mkdir timed out', + ); + }, { needsSandbox: true }); + + await probe('readdir', async () => { + await withTimeout( + sandbox!.filesystem.writeFile(`${dirPath}/file1.txt`, 'content1'), + FS_TIMEOUT_MS, + 'readdir setup writeFile timed out', + ); + await withTimeout( + sandbox!.filesystem.writeFile(`${dirPath}/file2.txt`, 'content2'), + FS_TIMEOUT_MS, + 'readdir setup writeFile timed out', + ); + const entries = (await withTimeout( + sandbox!.filesystem.readdir(dirPath), + FS_TIMEOUT_MS, + 'readdir timed out', + )) as FileEntry[]; + const names = entries.map((entry) => entry.name); + if (!names.includes('file1.txt') || !names.includes('file2.txt')) { + throw new Error('readdir did not return the expected files'); + } + }, { needsSandbox: true }); + + await probe('exists', async () => { + const fileExists = await withTimeout( + sandbox!.filesystem.exists(filePath), + FS_TIMEOUT_MS, + 'exists timed out', + ); + if (!fileExists) throw new Error('exists returned false for an existing file'); + }, { needsSandbox: true }); + + await probe('remove', async () => { + await withTimeout(sandbox!.filesystem.remove(filePath), FS_TIMEOUT_MS, 'remove timed out'); + const stillExists = await withTimeout( + sandbox!.filesystem.exists(filePath), + FS_TIMEOUT_MS, + 'exists after remove timed out', + ); + if (stillExists) throw new Error('remove did not delete the file'); + }, { needsSandbox: true }); + + await probe('snapshot.create', async () => { + if (!compute.snapshot) throw new Error('Snapshot manager not exposed'); + const snapshot = await withTimeout( + compute.snapshot.create(sandbox!.sandboxId, { name: `capability-snapshot-${Date.now()}` }), + SNAPSHOT_TIMEOUT_MS, + 'Snapshot creation timed out', + ); + createdSnapshotId = extractId(snapshot, 'snapshot'); + }, { needsSandbox: true }); + + await probe('snapshot.list', async () => { + if (!compute.snapshot) throw new Error('Snapshot manager not exposed'); + const list = await withTimeout(compute.snapshot.list(), SNAPSHOT_TIMEOUT_MS, 'Snapshot list timed out'); + if (!Array.isArray(list)) throw new Error('Snapshot list did not return an array'); + }); + + await probe('snapshot.delete', async () => { + if (!compute.snapshot) throw new Error('Snapshot manager not exposed'); + if (!createdSnapshotId) throw new Error('Snapshot creation failed, nothing to delete'); + await withTimeout( + compute.snapshot.delete(createdSnapshotId), + SNAPSHOT_TIMEOUT_MS, + 'Snapshot delete timed out', + ); + }); + + await probe('template.create', async () => { + if (!compute.template) throw new Error('Template manager not exposed'); + const template = await withTimeout( + compute.template.create({ name: `capability-template-${Date.now()}`, description: 'benchmark capability probe' }), + TEMPLATE_TIMEOUT_MS, + 'Template creation timed out', + ); + createdTemplateId = extractId(template, 'template'); + }); + + await probe('template.list', async () => { + if (!compute.template) throw new Error('Template manager not exposed'); + const list = await withTimeout(compute.template.list(), TEMPLATE_TIMEOUT_MS, 'Template list timed out'); + if (!Array.isArray(list)) throw new Error('Template list did not return an array'); + }); + + await probe('template.delete', async () => { + if (!compute.template) throw new Error('Template manager not exposed'); + if (!createdTemplateId) throw new Error('Template creation failed, nothing to delete'); + await withTimeout( + compute.template.delete(createdTemplateId), + TEMPLATE_TIMEOUT_MS, + 'Template delete timed out', + ); + }); + } finally { + await probe('destroy', async () => { + if (!sandbox) throw new Error('sandbox creation failed, no sandbox to destroy'); + await withTimeout( + sandbox.destroy(), + participant.destroyTimeoutMs ?? DESTROY_TIMEOUT_MS, + 'Destroy timed out', + ); + }); + } + + return { data: { features: matrix } as unknown as JsonObject }; +}); + +function extractId(resource: unknown, kind: 'snapshot' | 'template'): string { + if (resource && typeof resource === 'object') { + const record = resource as Record; + if (typeof record.id === 'string') return record.id; + if (kind === 'snapshot' && typeof record.snapshotId === 'string') return record.snapshotId; + if (kind === 'template' && typeof record.templateId === 'string') return record.templateId; + } + if (typeof resource === 'string') return resource; + throw new Error(`Could not determine ${kind} id from create response`); +} diff --git a/benchmarks/sandbox/types.ts b/benchmarks/sandbox/types.ts index 581d7844..f994e8bb 100644 --- a/benchmarks/sandbox/types.ts +++ b/benchmarks/sandbox/types.ts @@ -68,3 +68,19 @@ export interface StaggeredBenchmarkResult extends BenchmarkResult { /** Per-sandbox timing profile showing launch offset and TTI */ rampProfile: { launchedAt: number; readyAt: number; ttiMs: number }[]; } + +/** Result of probing a single ComputeSDK capability. */ +export interface CapabilityFeatureResult { + passed: boolean; + error?: string; +} + +/** Per-provider feature matrix for the sandbox capabilities benchmark. */ +export type ProviderCapabilityMatrix = Record; + +/** One provider's capability matrix result. */ +export interface CapabilityMatrixResult { + provider: string; + features: ProviderCapabilityMatrix; +} + diff --git a/package.json b/package.json index 69e64213..cd5a086a 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3", "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", + "bench:capabilities": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts", "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider blaxel", "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider codesandbox", "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider daytona", From 3465cb887518a1f8a370669e10063773f7c7b6a0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:22:27 +0000 Subject: [PATCH 02/10] Aggregate per-iteration capability matrices deterministically Co-Authored-By: Noah Kiser --- benchmarks/sandbox/capabilities-results.ts | 33 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/benchmarks/sandbox/capabilities-results.ts b/benchmarks/sandbox/capabilities-results.ts index 944eafac..62d95873 100644 --- a/benchmarks/sandbox/capabilities-results.ts +++ b/benchmarks/sandbox/capabilities-results.ts @@ -3,7 +3,8 @@ import os from 'node:os'; import path from 'node:path'; import type { ParticipantRecords, ResolvedRunConfig } from '@benchsdk/runner'; import type { TaskResultRecord } from '@benchsdk/client'; -import type { ProviderCapabilityMatrix, CapabilityMatrixResult } from './types.js'; +import type { ProviderCapabilityMatrix, CapabilityMatrixResult, CapabilityFeatureResult } from './types.js'; +import { byTaskIndex } from '../src/util/records.js'; const RESULTS_VERSION = '1.0'; @@ -29,14 +30,40 @@ function extractMatrix(record: TaskResultRecord): ProviderCapabilityMatrix { return matrix; } +/** + * Combine per-iteration feature matrices into a single provider matrix. + * + * A feature passes only when *every* iteration reports it passed. The error + * message comes from the first failing iteration, making the aggregate + * deterministic regardless of completion order. + */ +function aggregateFeatureMatrices(matrices: ProviderCapabilityMatrix[]): ProviderCapabilityMatrix { + const allKeys = new Set(); + for (const matrix of matrices) { + for (const key of Object.keys(matrix)) { + allKeys.add(key); + } + } + + const aggregate: ProviderCapabilityMatrix = {}; + for (const key of allKeys) { + const entries: (CapabilityFeatureResult | undefined)[] = matrices.map((matrix) => matrix[key]); + const passed = entries.every((entry) => entry?.passed === true); + const firstFailure = entries.find((entry) => entry && !entry.passed && entry.error); + aggregate[key] = passed ? { passed: true } : { passed: false, ...(firstFailure?.error ? { error: firstFailure.error } : {}) }; + } + return aggregate; +} + /** Aggregate per-provider capability matrices and write dated + latest JSON. */ export async function writeSandboxCapabilitiesResults( participants: ParticipantRecords[], opts: { resultsDir: string; runConfig: ResolvedRunConfig }, ): Promise { const results: CapabilityMatrixResult[] = participants.map((participant) => { - const record = participant.records[0]; - const features = record ? extractMatrix(record) : {}; + const orderedRecords = byTaskIndex(participant.records); + const matrices = orderedRecords.map(extractMatrix); + const features = aggregateFeatureMatrices(matrices); return { provider: participant.participant, features }; }); From 68a8cae9ad31f8fadb41b4e6f8f4ea1a04c5c54e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:33:15 +0000 Subject: [PATCH 03/10] feat: add sandbox capabilities matrix workflow Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 221 +++++++++++++++++++++ benchmarks/src/merge-results.ts | 2 + 2 files changed, 223 insertions(+) create mode 100644 .github/workflows/sandbox-capabilities.yml diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml new file mode 100644 index 00000000..b0e4bb0a --- /dev/null +++ b/.github/workflows/sandbox-capabilities.yml @@ -0,0 +1,221 @@ +name: Sandbox Capabilities Matrix + +on: + schedule: + - cron: '0 10 * * 1' # Weekly on Monday at 10am UTC / 5am CDT + workflow_dispatch: + inputs: + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + iterations: + description: 'Iterations per provider' + required: false + default: '1' + concurrency: + description: 'Concurrent sandboxes per provider' + required: false + default: '1' + publish: + description: 'Commit and push results to the repo' + required: false + default: true + type: boolean + +concurrency: + group: sandbox-capabilities + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + id-token: write + +jobs: + bench: + name: Capabilities ${{ matrix.provider }} + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + # runs-on: self-hosted + timeout-minutes: 30 + env: + SHOULD_RUN: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider }} + strategy: + fail-fast: false + matrix: + provider: + - archil + - arker + - beam + - blaxel + - cloud-run + - cloudflare + - codesandbox + - createos + - daytona + - declaw + - e2b + - hopx + - isorun + - lightning + - modal + - mosaic + - namespace + - northflank + - opencomputer + - runloop + - run-cloud + - sandbox0 + - sail + - sprites + - superserve + - tenki + - tensorlake + - upstash + - vercel + steps: + - uses: actions/checkout@v4 + if: env.SHOULD_RUN == 'true' + - uses: pnpm/action-setup@v4 + if: env.SHOULD_RUN == 'true' + - uses: actions/setup-node@v4 + if: env.SHOULD_RUN == 'true' + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@v0 + if: env.SHOULD_RUN == 'true' + - name: Install dependencies + if: env.SHOULD_RUN == 'true' + run: | + if [ "${{ github.event_name }}" = 'schedule' ]; then + pnpm update + else + pnpm install --frozen-lockfile + fi + - name: Clear stale results from checkout + if: env.SHOULD_RUN == 'true' + run: rm -rf results/ + - name: Run capabilities benchmark + if: env.SHOULD_RUN == 'true' + run: | + . benchmarks/scripts/load-vault-secrets.sh '.*' + + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ + --provider ${{ matrix.provider }} \ + --iterations ${{ github.event.inputs.iterations || '1' }} \ + --concurrency ${{ github.event.inputs.concurrency || '1' }} \ + --run-key "$RUN_KEY" + - name: Upload results + if: always() && env.SHOULD_RUN == 'true' + uses: actions/upload-artifact@v4 + with: + name: results-capabilities-${{ matrix.provider }} + path: results/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect Capabilities Results + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + # runs-on: self-hosted + needs: bench + if: always() + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - uses: namespacelabs/nscloud-setup@v0 + - name: Install dependencies + run: | + if [ "${{ github.event_name }}" = 'schedule' ]; then + pnpm update + else + pnpm install --frozen-lockfile + fi + - name: Download all results + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: results-capabilities-* + - name: Merge results + run: npx tsx benchmarks/src/merge-results.ts --input artifacts + - name: Write GitHub Step Summary + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const latestPath = path.join('results', 'sandbox-capabilities', 'latest.json'); + if (!fs.existsSync(latestPath)) { + core.summary.addHeading('Sandbox Capabilities Matrix', 2); + core.summary.addRaw('No capabilities results were generated.'); + await core.summary.write(); + return; + } + + const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + const results = data.results || []; + const featureNames = results.length > 0 ? Object.keys(results[0].features || {}) : []; + + const summary = core.summary; + summary.addHeading('Sandbox Capabilities Matrix', 2); + summary.addRaw(`

Timestamp: ${data.timestamp || 'unknown'}
Providers: ${results.length}

`); + + const header = ['Provider', ...featureNames]; + const rows = results.map(({ provider, features }) => { + return [ + provider, + ...featureNames.map((name) => (features && features[name] && features[name].passed ? '✅' : '❌')), + ]; + }); + summary.addTable([header, ...rows]); + + const failures = results + .map(({ provider, features }) => ({ + provider, + failed: Object.entries(features || {}) + .filter(([, value]) => !value.passed) + .map(([name, value]) => ({ name, error: value.error || 'failed' })), + })) + .filter((entry) => entry.failed.length > 0); + + if (failures.length > 0) { + summary.addHeading('Failure details', 3); + for (const { provider, failed } of failures) { + const content = failed + .map((failure) => `- **${failure.name}**: ${failure.error}`) + .join('\n'); + summary.addDetails(`${provider} (${failed.length} failures)`, content); + } + } + + await summary.write(); + - name: Commit and push + if: github.event.inputs.publish != 'false' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add results/sandbox-capabilities/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update sandbox capabilities results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1 diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index 793546c5..7741cd50 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -45,6 +45,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '../..'); const SANDBOX_WORKLOAD_DIRS = new Set([ 'sandbox-dax', + 'sandbox-capabilities', ]); const args = process.argv.slice(2); @@ -84,6 +85,7 @@ function modeToDir(mode: string): string { case 'burst': case 'concurrent': return 'burst_tti'; case 'sandbox-dax': return 'sandbox-dax'; + case 'sandbox-capabilities': return 'sandbox-capabilities'; default: return `${mode}_tti`; } } From 6373957f58e6126b910ab2988277058c215eeeb5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:41:59 +0000 Subject: [PATCH 04/10] fix(workflow): use frozen lockfile and provider-scoped vault secrets Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 48 ++++++++++++++++------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index b0e4bb0a..876eb11a 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -87,19 +87,46 @@ jobs: if: env.SHOULD_RUN == 'true' - name: Install dependencies if: env.SHOULD_RUN == 'true' - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi + run: pnpm install --frozen-lockfile - name: Clear stale results from checkout if: env.SHOULD_RUN == 'true' run: rm -rf results/ - name: Run capabilities benchmark if: env.SHOULD_RUN == 'true' run: | - . benchmarks/scripts/load-vault-secrets.sh '.*' + case '${{ matrix.provider }}' in + archil) KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) KEYS='ARKER_API_KEY' ;; + beam) KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) KEYS='CSB_API_KEY' ;; + createos) KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) KEYS='DAYTONA_API_KEY' ;; + declaw) KEYS='DECLAW_API_KEY' ;; + e2b) KEYS='E2B_API_KEY' ;; + hopx) KEYS='HOPX_API_KEY' ;; + isorun) KEYS='ISORUN_API_KEY' ;; + lightning) KEYS='LIGHTNING_API_KEY' ;; + modal) KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + mosaic) KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) KEYS='NSC_TOKEN' ;; + northflank) KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + runloop) KEYS='RUNLOOP_API_KEY' ;; + run-cloud) KEYS='RUN_CLOUD_API_KEY' ;; + sandbox0) KEYS='SANDBOX0_TOKEN' ;; + sail) KEYS='SAIL_API_KEY' ;; + sprites) KEYS='SPRITES_TOKEN' ;; + superserve) KEYS='SUPERSERVE_API_KEY' ;; + tenki) KEYS='TENKI_API_KEY' ;; + tensorlake) KEYS='TENSORLAKE_API_KEY' ;; + upstash) KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + esac + KEYS="${KEYS}|BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY" + . benchmarks/scripts/load-vault-secrets.sh "$KEYS" RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ @@ -133,12 +160,7 @@ jobs: cache: 'pnpm' - uses: namespacelabs/nscloud-setup@v0 - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = 'schedule' ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi + run: pnpm install --frozen-lockfile - name: Download all results uses: actions/download-artifact@v4 with: From 246bd5d7b1681a3228a5ad0b3e3e8c2967a2cad9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:52:52 +0000 Subject: [PATCH 05/10] fix(workflow): quote dispatch inputs, use pnpm exec, split permissions, avoid empty KEYS regex Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 28 ++++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 876eb11a..f5a36bcd 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -27,14 +27,14 @@ concurrency: group: sandbox-capabilities cancel-in-progress: true -permissions: - contents: write - pull-requests: write - id-token: write +permissions: {} jobs: bench: name: Capabilities ${{ matrix.provider }} + permissions: + contents: read + id-token: write runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe # runs-on: self-hosted timeout-minutes: 30 @@ -124,15 +124,20 @@ jobs: tensorlake) KEYS='TENSORLAKE_API_KEY' ;; upstash) KEYS='UPSTASH_BOX_API_KEY' ;; vercel) KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) KEYS='' ;; esac - KEYS="${KEYS}|BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY" + if [ -n "$KEYS" ]; then + KEYS="BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY|${KEYS}" + else + KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + fi . benchmarks/scripts/load-vault-secrets.sh "$KEYS" RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ - --provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || '1' }} \ - --concurrency ${{ github.event.inputs.concurrency || '1' }} \ + pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ + --provider '${{ matrix.provider }}' \ + --iterations '${{ github.event.inputs.iterations || '1' }}' \ + --concurrency '${{ github.event.inputs.concurrency || '1' }}' \ --run-key "$RUN_KEY" - name: Upload results if: always() && env.SHOULD_RUN == 'true' @@ -149,6 +154,9 @@ jobs: # runs-on: self-hosted needs: bench if: always() + permissions: + contents: write + id-token: write steps: - uses: actions/checkout@v4 with: @@ -167,7 +175,7 @@ jobs: path: artifacts/ pattern: results-capabilities-* - name: Merge results - run: npx tsx benchmarks/src/merge-results.ts --input artifacts + run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts - name: Write GitHub Step Summary uses: actions/github-script@v7 with: From f9e5a3267059ae6ec24fdc3be2e69c77466a0b26 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:25:01 +0000 Subject: [PATCH 06/10] ci: harden sandbox-capabilities workflow Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 94 ++++++++++++---------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index f5a36bcd..a74f91bf 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -74,16 +74,18 @@ jobs: - upstash - vercel steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: env.SHOULD_RUN == 'true' - - uses: pnpm/action-setup@v4 + with: + persist-credentials: false + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 if: env.SHOULD_RUN == 'true' - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' with: node-version: 24 cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 if: env.SHOULD_RUN == 'true' - name: Install dependencies if: env.SHOULD_RUN == 'true' @@ -94,42 +96,43 @@ jobs: - name: Run capabilities benchmark if: env.SHOULD_RUN == 'true' run: | + COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' case '${{ matrix.provider }}' in - archil) KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; - arker) KEYS='ARKER_API_KEY' ;; - beam) KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; - blaxel) KEYS='BL_API_KEY|BL_WORKSPACE' ;; - cloud-run) KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; - cloudflare) KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; - codesandbox) KEYS='CSB_API_KEY' ;; - createos) KEYS='CREATEOS_SANDBOX_API_KEY' ;; - daytona) KEYS='DAYTONA_API_KEY' ;; - declaw) KEYS='DECLAW_API_KEY' ;; - e2b) KEYS='E2B_API_KEY' ;; - hopx) KEYS='HOPX_API_KEY' ;; - isorun) KEYS='ISORUN_API_KEY' ;; - lightning) KEYS='LIGHTNING_API_KEY' ;; - modal) KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; - mosaic) KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; - namespace) KEYS='NSC_TOKEN' ;; - northflank) KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; - opencomputer) KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; - runloop) KEYS='RUNLOOP_API_KEY' ;; - run-cloud) KEYS='RUN_CLOUD_API_KEY' ;; - sandbox0) KEYS='SANDBOX0_TOKEN' ;; - sail) KEYS='SAIL_API_KEY' ;; - sprites) KEYS='SPRITES_TOKEN' ;; - superserve) KEYS='SUPERSERVE_API_KEY' ;; - tenki) KEYS='TENKI_API_KEY' ;; - tensorlake) KEYS='TENSORLAKE_API_KEY' ;; - upstash) KEYS='UPSTASH_BOX_API_KEY' ;; - vercel) KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; - *) KEYS='' ;; + archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; + arker) PROVIDER_KEYS='ARKER_API_KEY' ;; + beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; + blaxel) PROVIDER_KEYS='BL_API_KEY|BL_WORKSPACE' ;; + cloud-run) PROVIDER_KEYS='CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET' ;; + cloudflare) PROVIDER_KEYS='CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET' ;; + codesandbox) PROVIDER_KEYS='CSB_API_KEY' ;; + createos) PROVIDER_KEYS='CREATEOS_SANDBOX_API_KEY' ;; + daytona) PROVIDER_KEYS='DAYTONA_API_KEY' ;; + declaw) PROVIDER_KEYS='DECLAW_API_KEY' ;; + e2b) PROVIDER_KEYS='E2B_API_KEY' ;; + hopx) PROVIDER_KEYS='HOPX_API_KEY' ;; + isorun) PROVIDER_KEYS='ISORUN_API_KEY' ;; + lightning) PROVIDER_KEYS='LIGHTNING_API_KEY' ;; + modal) PROVIDER_KEYS='MODAL_TOKEN_ID|MODAL_TOKEN_SECRET' ;; + mosaic) PROVIDER_KEYS='MOSAIC_API_URL|MOSAIC_API_TOKEN' ;; + namespace) PROVIDER_KEYS='NSC_TOKEN' ;; + northflank) PROVIDER_KEYS='NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID' ;; + opencomputer) PROVIDER_KEYS='OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL' ;; + runloop) PROVIDER_KEYS='RUNLOOP_API_KEY' ;; + run-cloud) PROVIDER_KEYS='RUN_CLOUD_API_KEY' ;; + sandbox0) PROVIDER_KEYS='SANDBOX0_TOKEN' ;; + sail) PROVIDER_KEYS='SAIL_API_KEY' ;; + sprites) PROVIDER_KEYS='SPRITES_TOKEN' ;; + superserve) PROVIDER_KEYS='SUPERSERVE_API_KEY' ;; + tenki) PROVIDER_KEYS='TENKI_API_KEY' ;; + tensorlake) PROVIDER_KEYS='TENSORLAKE_API_KEY' ;; + upstash) PROVIDER_KEYS='UPSTASH_BOX_API_KEY' ;; + vercel) PROVIDER_KEYS='VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID' ;; + *) PROVIDER_KEYS='' ;; esac - if [ -n "$KEYS" ]; then - KEYS="BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY|${KEYS}" + if [ -n "$PROVIDER_KEYS" ]; then + KEYS="^(${COMMON_KEYS}|${PROVIDER_KEYS})=" else - KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + KEYS="^(${COMMON_KEYS})=" fi . benchmarks/scripts/load-vault-secrets.sh "$KEYS" @@ -141,7 +144,7 @@ jobs: --run-key "$RUN_KEY" - name: Upload results if: always() && env.SHOULD_RUN == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: results-capabilities-${{ matrix.provider }} path: results/ @@ -158,26 +161,28 @@ jobs: contents: write id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 cache: 'pnpm' - - uses: namespacelabs/nscloud-setup@v0 + - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 - name: Install dependencies run: pnpm install --frozen-lockfile - name: Download all results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts/ pattern: results-capabilities-* - name: Merge results + continue-on-error: true run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts - name: Write GitHub Step Summary - uses: actions/github-script@v7 + if: always() + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const fs = require('fs'); @@ -233,6 +238,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + mkdir -p results/sandbox-capabilities git add results/sandbox-capabilities/ git diff --cached --quiet && echo "No changes to commit" && exit 0 git commit -m "chore: update sandbox capabilities results [skip ci]" From 5060185439f273c90a5f5e24da4e63c4b178c856 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:31:47 +0000 Subject: [PATCH 07/10] ci: pin pnpm/action-setup to v4.4.0 SHA Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index a74f91bf..1f5599b8 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -78,7 +78,7 @@ jobs: if: env.SHOULD_RUN == 'true' with: persist-credentials: false - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 if: env.SHOULD_RUN == 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' @@ -164,7 +164,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 From 7d292ed52bf5612d8702979c5d1165e2b6a8a30a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:37:10 +0000 Subject: [PATCH 08/10] ci: revert pnpm/action-setup pin to v4 tag SHA Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 1f5599b8..a74f91bf 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -78,7 +78,7 @@ jobs: if: env.SHOULD_RUN == 'true' with: persist-credentials: false - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 if: env.SHOULD_RUN == 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.SHOULD_RUN == 'true' @@ -164,7 +164,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 From a8fd05d1a2f8757886dc7fec1056acbe8bdc2b38 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:42:30 +0000 Subject: [PATCH 09/10] ci: avoid template injection by passing inputs via env vars Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index a74f91bf..dfca2228 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -95,9 +95,13 @@ jobs: run: rm -rf results/ - name: Run capabilities benchmark if: env.SHOULD_RUN == 'true' + env: + ITERATIONS: ${{ github.event.inputs.iterations || '1' }} + CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }} + PROVIDER: ${{ matrix.provider }} run: | COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' - case '${{ matrix.provider }}' in + case "$PROVIDER" in archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; arker) PROVIDER_KEYS='ARKER_API_KEY' ;; beam) PROVIDER_KEYS='BEAM_TOKEN|BEAM_WORKSPACE_ID' ;; @@ -138,9 +142,9 @@ jobs: RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ - --provider '${{ matrix.provider }}' \ - --iterations '${{ github.event.inputs.iterations || '1' }}' \ - --concurrency '${{ github.event.inputs.concurrency || '1' }}' \ + --provider "$PROVIDER" \ + --iterations "$ITERATIONS" \ + --concurrency "$CONCURRENCY" \ --run-key "$RUN_KEY" - name: Upload results if: always() && env.SHOULD_RUN == 'true' From 6dff2403dbb90ce8a712dc4adb4cf12595f68226 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:48:42 +0000 Subject: [PATCH 10/10] ci: remove unneeded vault and id-token permissions from collect job Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index dfca2228..f270418b 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -157,13 +157,12 @@ jobs: collect: name: Collect Capabilities Results - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + runs-on: namespace-profile-default # runs-on: self-hosted needs: bench if: always() permissions: contents: write - id-token: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: