Refresh data #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Refresh data | |
| # Keeps published data fresh: pull all upstream sources, rebuild the graph, | |
| # package dist/, verify it, and publish a data-YYYYMMDD release when — and | |
| # only when — the snapshot_id actually changed (publish.sh enforces that). | |
| # | |
| # Trigger rationale (04:17 UTC daily): | |
| # - EPSS posts epss_scores-current.csv.gz shortly after 00:00 UTC, so the | |
| # daily scores have landed and settled by 04:00. | |
| # - CISA KEV is updated on US business days, typically 18:00-22:00 UTC, so | |
| # this run picks up the previous business day's additions in full. | |
| # - CVEList V5, OSV, nuclei-templates and PoC-in-GitHub push continuously; | |
| # any time of day is equally fresh for them. | |
| # - :17 rather than :00 avoids the top-of-hour scheduled-job queue on | |
| # GitHub-hosted runners, which can delay a :00 cron by 10-30 minutes. | |
| on: | |
| schedule: | |
| - cron: '17 4 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| publish: | |
| description: 'Publish a data-YYYYMMDD release if the snapshot changed' | |
| type: boolean | |
| default: true | |
| permissions: | |
| contents: write | |
| # Never let two refreshes race: they share dist/ semantics and the release tag. | |
| concurrency: | |
| group: refresh-data | |
| cancel-in-progress: false | |
| jobs: | |
| refresh: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| env: | |
| # cargo's built-in libgit2 transport ignores url.insteadOf and the SSH | |
| # config below; shelling out to the git CLI honors both. | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # The raw source tree is ~17 GB (OSV extracts to ~9 GB, cvelistV5 is | |
| # ~3.5 GB). A stock runner does not have room for that, so reclaim the | |
| # preinstalled toolchains we do not use (~30 GB). | |
| - name: Free disk space | |
| run: | | |
| df -h / | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ | |
| /usr/local/share/boost /usr/local/share/powershell \ | |
| /usr/share/swift /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| df -h / | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: cargo-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: cargo-${{ runner.os }}- | |
| # The deps.dev cache is expensive to build (two rate-limited API calls | |
| # per package) and cheap to store, so it persists across runs and is | |
| # topped up incrementally below. | |
| - name: Restore deps.dev cache | |
| id: deps-dev-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: research/downloads/deps-dev | |
| key: deps-dev- | |
| restore-keys: deps-dev- | |
| # vulngraph-engine is a git dependency on the private copyleftdev/vulngraph | |
| # repo. A job's GITHUB_TOKEN is scoped to its own repo and cannot read it, | |
| # so cargo authenticates with a read-only deploy key over SSH instead. | |
| - name: Authenticate cargo to the private engine repo | |
| env: | |
| ENGINE_SSH_KEY: ${{ secrets.ENGINE_SSH_KEY }} | |
| run: | | |
| mkdir -p ~/.ssh | |
| printf '%s\n' "$ENGINE_SSH_KEY" > ~/.ssh/engine_ed25519 | |
| chmod 600 ~/.ssh/engine_ed25519 | |
| ssh-keyscan -t ed25519 github.com >> ~/.ssh/known_hosts 2>/dev/null | |
| printf '%s\n' \ | |
| 'Host github.com' \ | |
| ' IdentityFile ~/.ssh/engine_ed25519' \ | |
| ' IdentitiesOnly yes' >> ~/.ssh/config | |
| # insteadOf matches on longest prefix, and ".../vulngraph" is a prefix | |
| # of ".../vulngraph-data" — the first rule alone would also rewrite this | |
| # repo's own remote to SSH and break the badge push in publish.sh. The | |
| # longer identity-mapped rule below pins vulngraph-data back to HTTPS. | |
| git config --global url."git@github.com:copyleftdev/vulngraph".insteadOf \ | |
| "https://github.com/copyleftdev/vulngraph" | |
| git config --global url."https://github.com/copyleftdev/vulngraph-data".insteadOf \ | |
| "https://github.com/copyleftdev/vulngraph-data" | |
| git ls-remote --get-url https://github.com/copyleftdev/vulngraph | |
| git ls-remote --get-url https://github.com/copyleftdev/vulngraph-data.git | |
| - name: Build pipeline binary | |
| run: cargo build --release | |
| # fetch-deps-dev.sh enumerates packages out of the *previous* graph, so | |
| # seed one from the last release. Without it the deps.dev step is a | |
| # no-op and the build silently loses its dependency edges. | |
| - name: Seed previous graph from latest release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p builds/vulngraph.db | |
| if gh release download --repo "$GITHUB_REPOSITORY" \ | |
| --pattern vulngraph-db.tar.gz --output /tmp/prev-db.tar.gz 2>/dev/null; then | |
| tar -xzf /tmp/prev-db.tar.gz -C builds/vulngraph.db | |
| rm -f /tmp/prev-db.tar.gz | |
| echo "Seeded previous graph: $(du -sh builds/vulngraph.db | cut -f1)" | |
| else | |
| echo "No previous release to seed from — deps.dev enumeration will be skipped." | |
| fi | |
| - name: Download sources | |
| run: ./scripts/download_sources.sh | |
| # The 1.2 GB archive is dead weight once extracted, and refresh.sh runs | |
| # with --rebuild-only so it will not be re-read. | |
| - name: Drop OSV archive | |
| run: | | |
| rm -f research/downloads/osv/all.zip | |
| du -sh research/downloads/* | sort -h | |
| df -h / | |
| # Time-boxed: a cold cache is ~20k packages at ~3/sec. Each run extends | |
| # the cache; a failure or timeout here is non-fatal by design. | |
| - name: Top up deps.dev cache | |
| continue-on-error: true | |
| run: | | |
| mkdir -p research/downloads/deps-dev | |
| timeout 25m ./scripts/fetch-deps-dev.sh \ | |
| research/downloads/deps-dev builds/vulngraph.db || \ | |
| echo "deps.dev top-up ended early (time-boxed) — cache retained." | |
| - name: Save deps.dev cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: research/downloads/deps-dev | |
| key: deps-dev-${{ github.run_id }} | |
| - name: Configure git identity | |
| run: | | |
| git config user.name 'github-actions[bot]' | |
| git config user.email '41898282+github-actions[bot]@users.noreply.github.com' | |
| # Sources are already current from the steps above, so --rebuild-only: | |
| # build -> export-demo -> package -> verify -> promote -> publish. | |
| # publish.sh no-ops when snapshot_id matches the latest release. | |
| - name: Build, package, verify and publish | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p logs | |
| ARGS="--rebuild-only" | |
| if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.publish }}" = "true" ]; then | |
| ARGS="$ARGS --publish" | |
| fi | |
| set -o pipefail | |
| ./scripts/refresh.sh $ARGS 2>&1 | tee logs/refresh.log | |
| - name: Summarize | |
| if: always() | |
| run: | | |
| { | |
| echo '## VulnGraph data refresh' | |
| if [ -f dist/manifest.json ]; then | |
| python3 - <<'PY' | |
| import json | |
| m = json.load(open('dist/manifest.json')) | |
| print(f"- snapshot: `{m['snapshot_id']}`") | |
| print(f"- graph: {m['node_count']:,} nodes / {m['edge_count']:,} edges") | |
| print(f"- engine: `{m['engine_rev']}` (format {m['format_version']})") | |
| print(f"- sources: {len(m.get('sources') or {})}") | |
| PY | |
| else | |
| echo '- no dist/manifest.json produced' | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload manifest and log | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: refresh-${{ github.run_id }} | |
| path: | | |
| dist/manifest.json | |
| logs/refresh.log | |
| if-no-files-found: warn | |
| retention-days: 14 |