ci.yml

# Armsmith CI — runs the hardware-free replay suite on native arm64 + x86.
# Labels ubuntu-24.04-arm / ubuntu-22.04-arm are the verified free public-repo
# arm64 runners (GitHub's hosted-runner documentation). Everything in this
# suite is replay/fixture-based, so it needs zero Arm-specific setup — the
# arm64 leg simply proves the package is arch-clean and doubles as the
# substrate for the S1+ live-bench jobs.
name: ci

on:
  push:
  pull_request:

# Cancel superseded runs on the same ref to save arm64 preview-runner minutes.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  # ── STAGE 2: Security gate — verified-secret scan across full git history ──
  secret-scan:
    name: "Stage 2 · Secret scanning (TruffleHog)"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0   # full history so rotated/committed secrets are caught
      - name: TruffleHog verified-secret scan
        uses: trufflesecurity/trufflehog@main
        with:
          # Lob is excluded, and only Lob: its detector matches `test_` followed by
          # 35 characters, which is the exact shape of a pytest function name
          # (test_verify_accepts_matching_trusted_key). It flagged 10 of our test
          # names and zero credentials. Armsmith calls no Lob API, so the detector
          # can only ever produce noise here. Every other detector stays on.
          extra_args: --only-verified --exclude-detectors=Lob

  # ── STAGE 1: Quality — lint, types, the full suite, e2e, schema, audit ────
  test:
    name: "Stage 1 · Quality (${{ matrix.runner }} · py${{ matrix.python-version }})"
    strategy:
      fail-fast: false
      matrix:
        runner: [ubuntu-24.04-arm, ubuntu-22.04-arm, ubuntu-latest]
        python-version: ["3.11", "3.12"]
    runs-on: ${{ matrix.runner }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install
        run: |
          python -m venv .venv
          .venv/bin/pip install -e '.[dev]'
      - name: Lint (ruff)
        run: .venv/bin/ruff check .
      - name: Types (mypy — advisory)
        run: .venv/bin/mypy src
        continue-on-error: true
      - name: Unit + replay suite (457 tests, 100% line coverage) with coverage
        run: .venv/bin/python -m pytest -q --cov=armsmith --cov-report=xml --cov-report=term-missing
      - name: Offline end-to-end (scan → gate → sign → verify)
        run: .venv/bin/python scripts/verify_offline.py
      - name: Reproduce-gate CI twin (armsmith ci — the drop-in Arm perf gate)
        run: .venv/bin/armsmith ci --replay fixtures/replays/scenario_ragserve
      - name: Report schema is valid JSON Schema
        run: .venv/bin/python -c "import json, jsonschema; jsonschema.Draft202012Validator.check_schema(json.load(open('schema/report.schema.json')))"
      # The schema declares $id https://armsmith.edycu.dev/schema/report.schema.json, so that URL
      # has to actually serve it — site/ is the Vercel output root. Copy, don't symlink: Vercel
      # would not follow one. This asserts the served copy never drifts from the packaged original.
      - name: Published schema matches the packaged one ($id must not lie)
        run: diff -u src/armsmith/schema/report.schema.json site/schema/report.schema.json
      - name: Dependency audit (pip-audit — advisory)
        run: |
          .venv/bin/pip install pip-audit
          .venv/bin/pip-audit --skip-editable
        continue-on-error: true

  # ── STAGE 3: Build verification ───────────────────────────────────────
  # Every other job installs with `pip install -e .`, which serves the rule
  # packs and the report schema straight out of the source tree. A published
  # wheel does not: if the YAML packs under rules/packs/ or schema/*.json are
  # not declared as package data, `pip install armsmith` yields a CLI that
  # imports fine and then cannot load a single rule. So the wheel is built and
  # exercised from a CLEAN venv, with the repo removed from sys.path.
  build:
    name: "Stage 3 · Build verification (wheel + sdist)"
    needs: [test]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - name: Build sdist + wheel
        run: |
          python -m pip install --upgrade build twine
          python -m build
          ls -la dist/
      - name: Metadata is valid for distribution
        run: python -m twine check dist/*
      - name: Install the wheel into a clean venv and exercise the CLI
        run: |
          python -m venv /tmp/clean
          /tmp/clean/bin/pip install dist/*.whl
          cd /tmp                      # out of the repo: nothing on sys.path to fall back to
          /tmp/clean/bin/armsmith version
          # Proves the 13 YAML rule packs shipped inside the wheel.
          /tmp/clean/bin/armsmith rules list | tee /tmp/rules.txt
          grep -c 'R1[0-3]\|R[1-9]' /tmp/rules.txt
          # Proves the JSON Schema shipped too — verify loads it by package path.
          /tmp/clean/bin/python -c "from armsmith.report import schema_path; p=schema_path(); print('schema:', p); assert p.exists()"
      - name: Upload the distributions
        uses: actions/upload-artifact@v7
        with:
          name: armsmith-dist
          path: dist/
          retention-days: 7

  # ── The live leg: a REAL Arm measurement, not a replay ────────────────
  # Compiles bench/int8_dot.c twice from one source (generic ARMv8.0 vs
  # ARMv8.2+dotprod — the exact flag rule R2 flags), disassembles both binaries
  # to count SDOT, ABAB-interleaves the timed runs, and puts the samples through
  # the same reproduce gate as every replay bundle. Runs on native arm64 because
  # there is nowhere else it *can* honestly run.
  live-bench:
    name: "Stage 5 · Live Arm reproduce gate (native arm64)"
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - name: Install
        run: |
          python -m venv .venv
          .venv/bin/pip install -e '.[dev]'
      - name: Toolchain provenance (recorded into the report)
        run: |
          gcc --version | head -1
          objdump --version | head -1
          lscpu | sed -n '1,12p'
      - name: Signing key for this run
        run: .venv/bin/armsmith keys init
      - name: Live A/B on real Arm silicon
        run: |
          .venv/bin/armsmith bench-live \
            --instance "github-hosted:ubuntu-24.04-arm" \
            --out report-live.json \
            --markdown live-evidence.md \
            --require-witness
      - name: Verify the live report (hash + ed25519 + schema + recompute)
        run: .venv/bin/armsmith verify report-live.json
      # Second live A/B, different ISA extension and different instruction, so
      # the live leg reads as a harness rather than one lucky microbenchmark.
      # SMMLA comes from an ACLE intrinsic behind the feature macro (GCC does
      # not reliably auto-vectorize this shape) — the source says so, and both
      # paths compute identical arithmetic so the gate would drop any divergence
      # on output-hash inequality.
      #
      # --require-witness is ON: run 31303... measured SMMLA 0 -> 1 on this
      # runner, so the instruction is proven to emit here. If a future
      # toolchain silently stops emitting it, the premise of this case is gone
      # and the build should say so rather than quietly measure nothing.
      - name: Live A/B #2 — SMMLA under +i8mm (native arm64)
        run: |
          .venv/bin/armsmith bench-live \
            --case mmla \
            --require-witness \
            --instance "github-hosted:ubuntu-24.04-arm" \
            --out report-live-mmla.json \
            --markdown live-evidence-mmla.md
          .venv/bin/armsmith verify report-live-mmla.json
          .venv/bin/python - <<'PY'
          import json
          r = json.load(open("report-live-mmla.json"))
          w = r["artifacts"]["isa_witness"]
          base = w["baseline"]["counts"].get("smmla", 0)
          cand = w["fix_i8mm"]["counts"].get("smmla", 0)
          print(f"SMMLA baseline={base} candidate={cand} delta={w['delta_total']}")
          print("verdict:", r["fixes"][0]["verdict"])
          PY
      # The record -> diagnose path on REAL hardware. Everywhere else in CI the
      # probe rules read fixtures we wrote; here `lscpu` and the THP sysfs node
      # genuinely exist, so this is the only job that can produce a bundle whose
      # manifest says "synthetic": false from observations nobody authored.
      # It proves the path a stranger uses on their own machine actually works
      # end to end, on the target ISA, at zero extra build cost.
      - name: Record a bundle from this Arm host, then diagnose with it
        run: |
          set -euo pipefail
          # Produce REAL instrument artifacts on this runner, then hand them to
          # `record`. Everything below is observed here — nothing is authored.
          #
          #   pip-install.log  -> R8 (sdist fallback for perf-critical wheels)
          #   numpy venv       -> R3 (which BLAS numpy actually linked)
          #   build.log        -> R2 (was the kernel built with -mcpu/-march?)
          #   lscpu / THP      -> captured by `record` itself
          python -m venv /tmp/probe-venv
          /tmp/probe-venv/bin/pip install -v numpy 2>&1 | tee pip-install.log | tail -3
          gcc -O3 -march=armv8-a -v -o /tmp/probe-kernel \
            src/armsmith/bench/int8_dot.c 2> build.log
          tail -2 build.log

          .venv/bin/armsmith record . \
            --out ./arm-bundle \
            --scenario "github-arm64-runner" \
            --python /tmp/probe-venv/bin/python \
            --pip-log pip-install.log \
            --build-log build.log \
            --note "captured on a GitHub-hosted ubuntu-24.04-arm runner"
          echo "--- manifest ---"
          cat ./arm-bundle/manifest.json

          .venv/bin/armsmith diagnose --replay ./arm-bundle \
            --out report-recorded.json --pr-dry-run
          .venv/bin/armsmith verify report-recorded.json

          # A green tick must not be reachable by a bundle that captured
          # nothing. Assert the bundle is real, that probe rules were actually
          # enabled by it, and that rules genuinely RAN rather than all skipping.
          .venv/bin/python - <<'PY'
          import json
          m = json.load(open("arm-bundle/manifest.json"))
          r = json.load(open("report-recorded.json"))
          assert m["synthetic"] is False, m
          got, enabled = set(m["captured_probes"]), set(m["rules_enabled"])
          assert "lscpu" in got, f"expected lscpu on an Arm runner, got {sorted(got)}"
          assert enabled, "bundle enabled no probe rules — the real-data path proved nothing"
          ran = [f for f in r["findings"] if f["status"] != "skipped"]
          assert len(ran) >= 4, f"only {len(ran)} rules ran with data: {[f['rule_id'] for f in ran]}"
          print("captured:", sorted(got))
          print("probe rules enabled:", sorted(enabled))
          print("rules that RAN on observed data:",
                {f["rule_id"]: f["status"] for f in ran})
          PY
      - name: Publish the evidence to the run summary
        if: always()
        run: |
          if [ -f live-evidence.md ]; then cat live-evidence.md >> "$GITHUB_STEP_SUMMARY"; fi
      - name: Upload the signed live report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: armsmith-live-report-arm64
          path: |
            report-live.json
            live-evidence.md
            report-recorded.json
            report-live-mmla.json
            live-evidence-mmla.md

  # ── STAGE 6: Deploy gate ──────────────────────────────────────────────
  # One required check that means "everything passed", so branch protection
  # and the release workflow have a single thing to depend on instead of
  # enumerating a matrix that changes shape whenever a runner is added.
  deploy-gate:
    name: "Stage 6 · Deploy gate"
    needs: [secret-scan, test, build, live-bench]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    steps:
      - name: All gates passed
        run: |
          {
            echo "## ✅ All CI gates passed"
            echo ""
            echo "| stage | what it proved |"
            echo "|---|---|"
            echo "| 1 · Quality | ruff + mypy + 457 tests at 100% line coverage, on 6 legs (4 native arm64) |"
            echo "| 2 · Security | TruffleHog verified-secret scan over full git history · pip-audit |"
            echo "| 3 · Build | wheel installs clean and still finds its rule packs + schema |"
            echo "| 4 · E2E | offline scan → gate → sign → verify, plus the \`armsmith ci\` twin |"
            echo "| 5 · Live Arm | a real Neoverse measurement through the same reproduce gate |"
          } >> "$GITHUB_STEP_SUMMARY"

  # TODO(S1): the remaining live instruments — hyperfine/llama-bench legs,
  # cosign keyless attestation of report.json (OIDC), and the noise-floor study
  # matrix across both arm labels.