levelcode/.github/workflows/eval-regression.yml at master · yethikrishna/levelcode · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# Eval Regression CI
# Runs buffbench on every PR against a reduced task set, compares to main branch
# baseline, and posts a comment showing regressions/improvements.

name: Eval Regression

on:
  pull_request:
    branches: ['master']
  workflow_dispatch:

concurrency:
  group: eval-regression-${{ github.ref }}
  cancel-in-progress: true

jobs:
  eval-regression:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout PR
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.3.5'

      - name: Cache dependencies
        uses: actions/cache@v4
        with:
          path: |
            node_modules
            */node_modules
            packages/*/node_modules
          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lock*') }}
          restore-keys: |
            ${{ runner.os }}-deps-

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Set environment variables
        env:
          SECRETS_CONTEXT: ${{ toJSON(secrets) }}
        run: |
          VAR_NAMES=$(bun scripts/generate-ci-env.ts)
          echo "$SECRETS_CONTEXT" | jq -r --argjson vars "$VAR_NAMES" '
            to_entries | .[] | select(.key as $k | $vars | index($k)) | .key + "=" + .value
          ' >> $GITHUB_ENV
          echo "LEVELCODE_GITHUB_ACTIONS=true" >> $GITHUB_ENV
          echo "LEVELCODE_GITHUB_TOKEN=${{ secrets.LEVELCODE_GITHUB_TOKEN }}" >> $GITHUB_ENV

      - name: Build SDK
        run: cd sdk && bun run build

      - name: Save baseline (main branch)
        id: baseline
        run: |
          git stash
          git checkout master
          mkdir -p evals/buffbench/logs-baseline
          cd evals
          bun run buffbench/main.ts --eval-file buffbench/eval-plane.json --agents base2 --concurrency 2 2>&1 | tee baseline-run.log || true
          cp -r buffbench/logs buffbench/logs-baseline/ 2>/dev/null || true
          git checkout ${{ github.head_ref || github.ref_name }}
          git stash pop 2>/dev/null || true

      - name: Run PR evals
        id: pr-run
        run: |
          mkdir -p evals/buffbench/logs-pr
          cd evals
          bun run buffbench/main.ts --eval-file buffbench/eval-plane.json --agents base2 --concurrency 2 2>&1 | tee pr-run.log || true
          cp -r buffbench/logs buffbench/logs-pr/ 2>/dev/null || true

      - name: Compare results
        id: compare
        run: |
          cd evals
          BASELINE=$(find buffbench/logs-baseline/logs -name "FINAL_RESULTS.json" -type f 2>/dev/null | head -1)
          CURRENT=$(find buffbench/logs-pr/logs -name "FINAL_RESULTS.json" -type f 2>/dev/null | head -1)
          if [ -z "$BASELINE" ] || [ -z "$CURRENT" ]; then
            echo "### ⚠️ Eval Run Incomplete" > $GITHUB_OUTPUT
            echo "Could not locate FINAL_RESULTS.json for both runs." >> $GITHUB_OUTPUT
            echo "" >> $GITHUB_OUTPUT
            echo "Baseline found: ${BASELINE:-none}" >> $GITHUB_OUTPUT
            echo "Current found: ${CURRENT:-none}" >> $GITHUB_OUTPUT
            echo "has_report=false" >> $GITHUB_OUTPUT
          else
            REPORT=$(bun run ci/eval-compare.ts "$BASELINE" "$CURRENT" --markdown 2>&1 || true)
            echo "report<<EOF" >> $GITHUB_OUTPUT
            echo "$REPORT" >> $GITHUB_OUTPUT
            echo "EOF" >> $GITHUB_OUTPUT
            echo "has_report=true" >> $GITHUB_OUTPUT
          fi

      - name: Find existing comment
        id: find-comment
        uses: peter-evans/find-comment@v3
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: 'Eval Regression Report'

      - name: Post or update PR comment
        uses: peter-evans/create-or-update-comment@v4
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ${{ steps.compare.outputs.report }}

            ---
            _Auto-generated by eval-regression workflow for commit ${{ github.event.pull_request.head.sha }} (vs master baseline)_
          edit-mode: replace