docs(examples): add benchSDK example benchmarks

benchmarks/examples/00-hello-world.bench.ts

@@ -0,0 +1,75 @@
+/**
+ * Hello-world benchmark.
+ *
+ * This is the smallest possible benchSDK benchmark. It demonstrates:
+ *   - exporting a `config` created with `defineBenchmarkConfig`
+ *   - exporting a `task` created with `defineTask`
+ *   - a single participant
+ *   - three named steps (create, exec, destroy)
+ *   - attaching metrics with `measure()`
+ *   - writing to the worker log artifact with `log()`
+ *
+ * Run with:
+ *   bench run benchmarks/examples/00-hello-world.bench.ts --iterations 3
+ */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import type { NoopParticipant } from './participants.js';
+import { createNoopParticipant } from './participants.js';
+
+/**
+ * The benchmark config declares the platform identity and orchestration knobs.
+ *
+ * - `benchmarkSlug` is the URL-safe identifier used for the platform API and
+ *   dashboard (`.../benchmarks/examples-hello-world/runs/...`).
+ * - `benchmarkName` is the human-readable name shown in the dashboard.
+ * - `iterations` is the total number of times the task runs per participant.
+ * - `concurrency` is the maximum number of tasks in flight at once for a single
+ *   participant worker.
+ * - `participants` lists the providers to benchmark; the same task runs for each.
+ */
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'examples-hello-world',
+  benchmarkName: 'Examples: Hello World',
+  iterations: 3,
+  concurrency: 1,
+  participants: [createNoopParticipant('noop', 100)],
+});
+
+/**
+ * The task is the per-iteration workload.
+ *
+ * `defineTask<NoopParticipant>` tells TypeScript the participant type so the
+ * task body can access `participant.createCompute()`. The context also gives
+ * `step`, `measure`, and `log`.
+ */
+export const task = defineTask<NoopParticipant>(async ({ participant, step, measure, log }) => {
+  // `log` appends a free-form line to the worker log artifact. It is useful for
+  // narrating what the task is doing; the return value is not recorded.
+  log('starting hello-world iteration');
+
+  // `participant.createCompute()` is specific to the mock provider in this
+  // example; a real benchmark would call the provider SDK.
+  const compute = participant.createCompute();
+
+  // We capture our own start time because the runner's wall-clock timing
+  // includes the `destroy` step; for TTI we want create-through-first-command.
+  const start = performance.now();
+
+  // `step(name, fn)` runs `fn`, returns its value, and records a step record
+  // with timing, status, and any data measured inside it.
+  const sandbox = await step('create', () => compute.sandbox.create());
+  try {
+    const result = await step('exec', () => sandbox.runCommand('node -v'));
+
+    // `measure(data)` merges JSON into the currently active step, or into the
+    // task record if called outside a step. Here we attach the TTI and exit code
+    // to the task record's `data`.
+    measure({ ttiMs: performance.now() - start, exitCode: result.exitCode });
+  } finally {
+    // `reportConcurrency: false` tells the worker not to count this step in its
+    // concurrency heartbeat samples; cleanup steps typically do not run while
+    // other tasks are also being launched.
+    await step('destroy', () => sandbox.destroy(), { reportConcurrency: false });
+  }
+});

benchmarks/examples/01-multiple-providers.bench.ts

@@ -0,0 +1,57 @@
+/**
+ * Multi-provider benchmark.
+ *
+ * Demonstrates running the same task against several participants. The runner
+ * first filters out any participants whose `requiredEnvVars` are missing, then
+ * executes the task once per selected provider.
+ *
+ * Run with all providers:
+ *   bench run benchmarks/examples/01-multiple-providers.bench.ts --iterations 5 --concurrency 2
+ *
+ * Run with a subset:
+ *   bench run benchmarks/examples/01-multiple-providers.bench.ts --iterations 5 --provider alpha,beta
+ */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import type { NoopParticipant } from './participants.js';
+import { exampleProviders } from './participants.js';
+
+/**
+ * The config reuses `exampleProviders` from `participants.ts`. Each provider
+ * gets its own worker on the platform, and the same `task` is invoked for
+ * every iteration of every provider.
+ *
+ * - `iterations: 5` means each provider runs the task 5 times.
+ * - `concurrency: 2` means up to 2 of those tasks are in flight at once for a
+ *   given provider's worker.
+ * - `--provider alpha,beta` would limit the run to just those two names.
+ */
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'examples-multiple-providers',
+  benchmarkName: 'Examples: Multiple Providers',
+  iterations: 5,
+  concurrency: 2,
+  participants: exampleProviders,
+});
+
+/**
+ * The task body is identical to the hello-world example, but `participant`
+ * changes per worker: the runner first completes all iterations for `alpha`,
+ * then `beta`, then `gamma` (default `groupBy: 'participant'`).
+ */
+export const task = defineTask<NoopParticipant>(async ({ participant, step, measure, log }) => {
+  // Because the same task runs for every provider, logging the provider name
+  // makes the worker log easy to read.
+  log(`running on ${participant.name}`);
+
+  const compute = participant.createCompute();
+  const start = performance.now();
+
+  const sandbox = await step('create', () => compute.sandbox.create());
+  try {
+    const result = await step('exec', () => sandbox.runCommand('node -v'));
+    measure({ ttiMs: performance.now() - start, exitCode: result.exitCode });
+  } finally {
+    await step('destroy', () => sandbox.destroy(), { reportConcurrency: false });
+  }
+});

benchmarks/examples/02-phases.bench.ts

@@ -0,0 +1,66 @@
+/**
+ * Phases benchmark.
+ *
+ * Demonstrates `phases`: an ordered list of named segments, each with its own
+ * iteration count. The runner tags every task record with `data.phase` and the
+ * task receives `ctx.phase`, so the same task function can branch on the phase
+ * without doing index arithmetic.
+ *
+ * Run with:
+ *   bench run benchmarks/examples/02-phases.bench.ts --concurrency 2
+ */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import type { NoopParticipant } from './participants.js';
+import { createNoopParticipant } from './participants.js';
+
+/**
+ * `phases` and `iterations` are mutually exclusive in `defineBenchmarkConfig`.
+ * When `phases` is set, the total number of task slots is the sum of all phase
+ * iteration counts (here 2 + 4 = 6). The slots run in phase order: first every
+ * `cold` slot, then every `warm` slot.
+ *
+ * Each slot is tagged with its phase name, and `ctx.phase` inside the task is
+ * set to that name.
+ */
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'examples-phases',
+  benchmarkName: 'Examples: Phases',
+  phases: [
+    { name: 'cold', iterations: 2 },
+    { name: 'warm', iterations: 4 },
+  ],
+  concurrency: 2,
+  participants: [createNoopParticipant('noop', 100)],
+});
+
+/**
+ * The task uses `ctx.phase` to vary behavior. In a real benchmark this might
+ * mean using a cold-start payload for the `cold` phase and a warm-start payload
+ * for the `warm` phase. The phase name is also written into the measured data
+ * so the dashboard can group or filter by phase.
+ */
+export const task = defineTask<NoopParticipant>(async ({ participant, step, measure, log, phase }) => {
+  log(`starting ${phase ?? 'unknown'} phase`);
+
+  const compute = participant.createCompute();
+  const start = performance.now();
+
+  // In a real benchmark you might use a different payload per phase.
+  const command = phase === 'cold' ? 'node -v' : 'node -e "console.log(1+1)"';
+
+  const sandbox = await step('create', () => compute.sandbox.create());
+  try {
+    const result = await step('exec', () => sandbox.runCommand(command));
+
+    // `measure` expects a `JsonObject`; `phase` may be `undefined` in a config
+    // that does not use phases, so we only add it when it is present.
+    measure({
+      ttiMs: performance.now() - start,
+      exitCode: result.exitCode,
+      ...(phase ? { phase } : {}),
+    });
+  } finally {
+    await step('destroy', () => sandbox.destroy(), { reportConcurrency: false });
+  }
+});

benchmarks/examples/03-round-robin.bench.ts

@@ -0,0 +1,58 @@
+/**
+ * Round-robin benchmark.
+ *
+ * Demonstrates `groupBy: 'round'`. In this mode every participant runs its Nth
+ * task before any participant starts its (N+1)th, so the Nth tasks of all
+ * providers happen back-to-back under the same conditions. The runner builds
+ * the task records manually and streams them to the platform via a reporter.
+ *
+ * Run with:
+ *   bench run benchmarks/examples/03-round-robin.bench.ts --iterations 4 --concurrency 1 --group-by round
+ */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import type { NoopParticipant } from './participants.js';
+import { exampleProviders } from './participants.js';
+
+/**
+ * `groupBy: 'round'` changes execution order:
+ *
+ *   Round 0: alpha task 0, beta task 0, gamma task 0
+ *   Round 1: alpha task 1, beta task 1, gamma task 1
+ *   ...
+ *
+ * This is useful when you want the Nth iteration of every provider to start at
+ * roughly the same wall-clock time, rather than finishing all iterations for
+ * one provider before moving to the next.
+ *
+ * `concurrency` is set to 1 here because the round-robin path runs one task per
+ * round; the ordering is the primary concern, not per-participant burst.
+ */
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'examples-round-robin',
+  benchmarkName: 'Examples: Round Robin',
+  iterations: 4,
+  concurrency: 1,
+  groupBy: 'round',
+  participants: exampleProviders,
+});
+
+/**
+ * `taskIndex` is the zero-based slot index within the participant's assignment.
+ * In round mode it effectively represents the round number, so we log it to
+ * make the ordering visible in the worker log.
+ */
+export const task = defineTask<NoopParticipant>(async ({ participant, step, measure, log, taskIndex }) => {
+  log(`round ${taskIndex + 1} for ${participant.name}`);
+
+  const compute = participant.createCompute();
+  const start = performance.now();
+
+  const sandbox = await step('create', () => compute.sandbox.create());
+  try {
+    const result = await step('exec', () => sandbox.runCommand('node -v'));
+    measure({ ttiMs: performance.now() - start, exitCode: result.exitCode });
+  } finally {
+    await step('destroy', () => sandbox.destroy(), { reportConcurrency: false });
+  }
+});

benchmarks/examples/04-shapes.bench.ts

@@ -0,0 +1,69 @@
+/**
+ * Shapes benchmark.
+ *
+ * Demonstrates `shapes`: named variants of the same benchmark that swap the
+ * platform slug/name and any stable distinguishing knob (e.g. stagger delay).
+ * Scale knobs such as `iterations` and `concurrency` are still overridden from
+ * the CLI, so one file can back several platform benchmarks.
+ *
+ * Run the quick variant:
+ *   bench run benchmarks/examples/04-shapes.bench.ts --shape quick --iterations 2 --concurrency 1
+ *
+ * Run the thorough variant:
+ *   bench run benchmarks/examples/04-shapes.bench.ts --shape thorough --iterations 10 --concurrency 3
+ */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import type { NoopParticipant } from './participants.js';
+import { createNoopParticipant } from './participants.js';
+
+/**
+ * The base config defines the default identity. `shapes` then declares variants.
+ *
+ * - `quick` reports under a different platform slug and has no stagger delay.
+ * - `thorough` reports under another slug and staggers each task start by 250ms.
+ *
+ * `--shape quick` swaps the identity before the run is created, so the platform
+ * records the run under `examples-shapes-quick` instead of `examples-shapes`.
+ *
+ * `iterations` and `concurrency` are not part of the shape because they are
+ * environment-specific scale knobs; override them from the CLI per run.
+ */
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'examples-shapes',
+  benchmarkName: 'Examples: Shapes',
+  iterations: 3,
+  concurrency: 1,
+  shapes: {
+    quick: {
+      slug: 'examples-shapes-quick',
+      name: 'Examples: Shapes (Quick)',
+      staggerDelayMs: 0,
+    },
+    thorough: {
+      slug: 'examples-shapes-thorough',
+      name: 'Examples: Shapes (Thorough)',
+      staggerDelayMs: 250,
+    },
+  },
+  participants: [createNoopParticipant('noop', 100)],
+});
+
+/**
+ * The task itself is the same for both shapes; the only difference is the
+ * platform identity and the stagger delay selected by `--shape`.
+ */
+export const task = defineTask<NoopParticipant>(async ({ participant, step, measure, log }) => {
+  log('starting shaped iteration');
+
+  const compute = participant.createCompute();
+  const start = performance.now();
+
+  const sandbox = await step('create', () => compute.sandbox.create());
+  try {
+    const result = await step('exec', () => sandbox.runCommand('node -v'));
+    measure({ ttiMs: performance.now() - start, exitCode: result.exitCode });
+  } finally {
+    await step('destroy', () => sandbox.destroy(), { reportConcurrency: false });
+  }
+});