feat: add storage concurrency benchmark

benchmarks/storage/providers.test.ts

@@ -0,0 +1,9 @@
+import assert from 'node:assert/strict';
+import { normalizeGcsPrivateKey } from './providers.js';
+
+const pemBody = 'abc\\ndef';
+assert.equal(normalizeGcsPrivateKey(pemBody), 'abc\ndef');
+assert.equal(normalizeGcsPrivateKey(JSON.stringify(pemBody)), 'abc\ndef');
+assert.equal(normalizeGcsPrivateKey('abc\r\ndef'), 'abc\ndef');
+
+console.log('Storage provider credential normalization checks passed');

benchmarks/storage/providers.ts

@@ -10,6 +10,35 @@ import { archil } from '@storagesdk/adapters/archil';
 import { neon } from '@storagesdk/adapters/neon';
 import type { StorageProviderConfig } from './types.js';
 
+export function normalizeGcsPrivateKey(value: string): string {
+  let key = value.trim();
+  if (key.startsWith('"') && key.endsWith('"')) {
+    try {
+      key = JSON.parse(key) as string;
+    } catch {
+      // Fall through and normalize the value as plain text.
+    }
+  }
+  return key
+    .replaceAll('\\\\r\\\\n', '\r\n')
+    .replaceAll('\\n', '\n')
+    .replaceAll('\\r\\n', '\r\n')
+    .replace(/\r\n/g, '\n');
+}
+
+export function isStorageProviderAvailable(provider: StorageProviderConfig): boolean {
+  if (provider.requiredEnvVars.some((name) => !process.env[name])) {
+    return false;
+  }
+  try {
+    provider.createStorage();
+    return true;
+  } catch {
+    console.warn(`Skipping ${provider.name}: storage credentials could not be initialized`);
+    return false;
+  }
+}
+
 /**
  * Storage provider benchmark configurations.
  *
@@ -153,8 +182,7 @@ export const storageProviders: StorageProviderConfig[] = [
         projectId: process.env.GCS_PROJECT_ID!,
         credentials: {
           client_email: process.env.GCS_CLIENT_EMAIL!,
-          // Secrets store the key with literal "\n"; restore real newlines.
-          private_key: process.env.GCS_PRIVATE_KEY!.replace(/\\n/g, '\n'),
+          private_key: normalizeGcsPrivateKey(process.env.GCS_PRIVATE_KEY!),
         },
       }),
     }),

benchmarks/storage/report-storage-concurrency.ts

@@ -0,0 +1,114 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  STORAGE_CONCURRENCY_RESULTS_PATH,
+  type StorageConcurrencyResults,
+} from './storage-concurrency-results.js';
+
+const root = path.resolve('.');
+const input = process.env.STORAGE_CONCURRENCY_RESULTS
+  ? path.resolve(process.env.STORAGE_CONCURRENCY_RESULTS)
+  : STORAGE_CONCURRENCY_RESULTS_PATH;
+const data = JSON.parse(fs.readFileSync(input, 'utf8')) as StorageConcurrencyResults;
+
+function providerName(provider: string): string {
+  return {
+    'aws-s3': 'AWS S3',
+    'cloudflare-r2': 'Cloudflare R2',
+    'vercel-blob': 'Vercel Blob',
+    'azure-blob': 'Azure Blob',
+  }[provider] ?? provider;
+}
+
+function number(value: number | null): string {
+  return value === null ? '—' : value.toFixed(1);
+}
+
+function escapeXml(value: string): string {
+  return value.replace(/[<>&"']/g, (character) => ({
+    '<': '&lt;',
+    '>': '&gt;',
+    '&': '&amp;',
+    '"': '&quot;',
+    "'": '&apos;',
+  }[character]!));
+}
+
+const rows = data.results.flatMap((provider) =>
+  provider.cells.map((cell) => ({
+    provider: providerName(provider.provider),
+    ...cell,
+  })),
+);
+
+const summaries = [...data.results]
+  .sort((a, b) => b.compositeScore - a.compositeScore)
+  .map((provider) => ({
+    ...provider,
+    displayName: providerName(provider.provider),
+    peakThroughput: Math.max(...provider.cells.map((cell) => cell.throughputOpsPerSecond), 0),
+  }));
+
+const markdown = [
+  '# Storage Concurrency Results',
+  '',
+  `Run: \`${data.runId}\`  `,
+  `Generated: ${data.timestamp}`,
+  '',
+  '## Provider ranking',
+  '',
+  ...(summaries.length > 0
+    ? [`Top provider: **${summaries[0].displayName}** (${summaries[0].compositeScore.toFixed(2)}/100)`, '']
+    : ['No provider results were available.', '']),
+  '| Rank | Provider | Composite score | Success rate | Valid cells | Peak throughput (ops/s) |',
+  '|---:|---|---:|---:|---:|---:|',
+  ...summaries.map((provider, index) =>
+    `| ${index + 1} | ${provider.displayName} | ${provider.compositeScore.toFixed(2)} | ${(provider.successRate * 100).toFixed(1)}% | ${(provider.validCellRate * 100).toFixed(1)}% | ${provider.peakThroughput.toFixed(1)} |`,
+  ),
+  '',
+  'Scores combine throughput (45%), p50 latency (20%), p95 latency (20%), and p99 latency (15%) per cell, then apply success-rate penalties.',
+  '',
+  '## Cell details',
+  '',
+  '| Provider | Cell | Throughput (ops/s) | p50 (ms) | p95 (ms) | p99 (ms) | Success | Valid |',
+  '|---|---|---:|---:|---:|---:|---:|:---:|',
+  ...rows.map((row) =>
+    `| ${row.provider} | ${row.phase} | ${row.throughputOpsPerSecond.toFixed(1)} | ${number(row.p50Ms)} | ${number(row.p95Ms)} | ${number(row.p99Ms)} | ${(row.successRate * 100).toFixed(1)}% | ${row.valid ? 'yes' : 'NO'} |`,
+  ),
+  '',
+  'This is a closed-loop benchmark. Throughput is the output observed at the requested client concurrency.',
+].join('\n');
+
+fs.writeFileSync(path.join(root, 'storage-concurrency.md'), `${markdown}\n`);
+
+const width = 1200;
+const rowHeight = 28;
+const headerHeight = 70;
+const chartHeight = Math.max(180, summaries.length * rowHeight + 90);
+const barWidth = 700;
+const chartRows = summaries.map((row, index) => {
+  const y = headerHeight + index * rowHeight;
+  const widthValue = Math.max(1, (row.compositeScore / 100) * barWidth);
+  return `
+  <text x="12" y="${y + 18}" class="label">${escapeXml(row.displayName)}</text>
+  <rect x="390" y="${y + 4}" width="${widthValue.toFixed(1)}" height="18" class="${row.validCellRate === 1 ? 'bar' : 'invalid'}"/>
+  <text x="${Math.min(1100, 400 + widthValue + 8)}" y="${y + 18}" class="value">${row.compositeScore.toFixed(2)}</text>`;
+}).join('');
+
+const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${chartHeight}" viewBox="0 0 ${width} ${chartHeight}">
+<style>
+.title { font: 700 24px sans-serif; fill: #24292f; }
+.subtitle { font: 13px sans-serif; fill: #57606a; }
+.label { font: 12px sans-serif; fill: #24292f; }
+.value { font: 12px sans-serif; fill: #57606a; }
+.bar { fill: #0969da; }
+.invalid { fill: #cf222e; }
+</style>
+<rect width="100%" height="100%" fill="white"/>
+<text x="12" y="30" class="title">Storage Concurrency Composite Score</text>
+<text x="12" y="50" class="subtitle">Run ${escapeXml(data.runId)} · higher is better</text>
+${chartRows}
+</svg>`;
+
+fs.writeFileSync(path.join(root, 'storage-concurrency.svg'), svg);
+console.log('Wrote storage-concurrency.md and storage-concurrency.svg');

benchmarks/storage/storage-concurrency-corpus.test.ts

@@ -0,0 +1,28 @@
+import assert from 'node:assert/strict';
+import { requestKey } from './storage-concurrency-corpus.js';
+
+function prefix(key: string): string {
+  return key.split('/')[2]!;
+}
+
+const workerZeroKeys = Array.from(
+  { length: 32 },
+  (_, opSeq) => requestKey(0, opSeq, 'SINGLE_PREFIX'),
+);
+const workerOneKeys = Array.from(
+  { length: 32 },
+  (_, opSeq) => requestKey(1, opSeq, 'SINGLE_PREFIX'),
+);
+
+assert.ok(workerZeroKeys.every((key) => prefix(key) === 'p00'));
+assert.ok(workerOneKeys.every((key) => prefix(key) === 'p00'));
+assert.notDeepEqual(workerZeroKeys, workerOneKeys);
+
+const singlePrefixKeys = new Set(
+  Array.from({ length: 128 }, (_, workerId) =>
+    requestKey(workerId, 0, 'SINGLE_PREFIX'),
+  ),
+);
+assert.equal(singlePrefixKeys.size, 128);
+
+console.log('Storage concurrency corpus key distribution checks passed');

benchmarks/storage/storage-concurrency-corpus.ts

@@ -0,0 +1,36 @@
+export const OBJECT_COUNT = 10_000;
+export const PREFIX_COUNT = 64;
+export const OBJECT_SIZE_BYTES = 1024;
+export const KEY_STRIDE = 7919;
+export const WORKER_PRIME = 104_729;
+const SINGLE_PREFIX_OBJECT_COUNT = Math.ceil(OBJECT_COUNT / PREFIX_COUNT);
+
+export type KeyDistribution = 'SINGLE_PREFIX' | 'SPREAD_64';
+
+export function corpusKey(index: number): string {
+  if (!Number.isInteger(index) || index < 0 || index >= OBJECT_COUNT) {
+    throw new Error(`Corpus index must be between 0 and ${OBJECT_COUNT - 1}`);
+  }
+  const prefix = index % PREFIX_COUNT;
+  return `bench/v1/p${prefix.toString().padStart(2, '0')}/obj${index.toString().padStart(6, '0')}`;
+}
+
+export function requestKey(
+  workerId: number,
+  opSeq: number,
+  distribution: KeyDistribution,
+): string {
+  const rawIndex = distribution === 'SINGLE_PREFIX'
+    ? (workerId * WORKER_PRIME + opSeq * KEY_STRIDE) % SINGLE_PREFIX_OBJECT_COUNT
+    : (workerId * WORKER_PRIME + opSeq * KEY_STRIDE) % OBJECT_COUNT;
+  const index = distribution === 'SINGLE_PREFIX' ? rawIndex * PREFIX_COUNT : rawIndex;
+  return corpusKey(index);
+}
+
+/** Stable 1 KB body shared by every seeded object. */
+export function corpusBody(): Uint8Array {
+  return Uint8Array.from(
+    { length: OBJECT_SIZE_BYTES },
+    (_, index) => (index * 31 + 17) % 256,
+  );
+}

benchmarks/storage/storage-concurrency-results.ts

@@ -0,0 +1,90 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import type { BenchmarkRunOutcome } from '@benchsdk/runner';
+import { scoreStorageConcurrencyProvider } from './storage-concurrency-scoring.js';
+
+export const STORAGE_CONCURRENCY_RESULTS_DIR = path.resolve('results/storage-concurrency');
+export const STORAGE_CONCURRENCY_RESULTS_PATH = path.join(
+  STORAGE_CONCURRENCY_RESULTS_DIR,
+  'latest.json',
+);
+
+export interface StorageConcurrencyCellResult {
+  phase: string;
+  operations: number;
+  measuredOperations: number;
+  concurrency: number;
+  keyDistribution: 'SINGLE_PREFIX' | 'SPREAD_64';
+  throughputOpsPerSecond: number;
+  p50Ms: number | null;
+  p95Ms: number | null;
+  p99Ms: number | null;
+  successRate: number;
+  throttleRate: number;
+  timeoutRate: number;
+  connectionErrorRate: number;
+  notFoundRate: number;
+  serverErrorRate: number;
+  clientErrorRate: number;
+  valid: boolean;
+  maxActiveRequests: number;
+}
+
+export interface StorageConcurrencyProviderResult {
+  provider: string;
+  cells: StorageConcurrencyCellResult[];
+  compositeScore: number;
+  successRate: number;
+  validCellRate: number;
+}
+
+export interface StorageConcurrencyResults {
+  version: '1.0';
+  timestamp: string;
+  runId: string;
+  environment: {
+    node: string;
+    platform: string;
+    arch: string;
+  };
+  results: StorageConcurrencyProviderResult[];
+}
+
+function isCellResult(value: unknown): value is StorageConcurrencyCellResult {
+  return typeof value === 'object' && value !== null
+    && typeof (value as { phase?: unknown }).phase === 'string'
+    && typeof (value as { concurrency?: unknown }).concurrency === 'number'
+    && typeof (value as { throughputOpsPerSecond?: unknown }).throughputOpsPerSecond === 'number';
+}
+
+export function writeStorageConcurrencyResults(
+  outcome: BenchmarkRunOutcome,
+): void {
+  const results: StorageConcurrencyProviderResult[] = outcome.participants.map((participant) => {
+    const result = {
+      provider: participant.participant,
+      cells: participant.records.flatMap((record) => {
+        const data = record.data;
+        return isCellResult(data) ? [data] : [];
+      }),
+    };
+    return { ...result, ...scoreStorageConcurrencyProvider(result) };
+  });
+
+  const output: StorageConcurrencyResults = {
+    version: '1.0',
+    timestamp: new Date().toISOString(),
+    runId: outcome.runId,
+    environment: {
+      node: process.version,
+      platform: os.platform(),
+      arch: os.arch(),
+    },
+    results,
+  };
+
+  fs.mkdirSync(STORAGE_CONCURRENCY_RESULTS_DIR, { recursive: true });
+  fs.writeFileSync(STORAGE_CONCURRENCY_RESULTS_PATH, `${JSON.stringify(output, null, 2)}\n`);
+  console.log(`Storage concurrency results written to ${STORAGE_CONCURRENCY_RESULTS_PATH}`);
+}