Skip to content

feat: Scope local-clone scanners to the core-repos allowlist #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rubambiza merged 2 commits into from
Jul 28, 2026
Merged

feat: Scope local-clone scanners to the core-repos allowlist #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d107ad
feat: Scope local-clone scanners to the core-repos allowlist
rubambiza Jul 27, 2026
45c11fa
style: Normalize SEEN_CANON casing in dep-bump-fixer
rubambiza Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions scripts/dep-bump-fixer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,25 @@ if [ ! -f "$REPORTS_DIR/baseline.json" ]; then

# Query merged Dependabot PRs across all repos (last 90 days)
: > "$TMPDIR/merged_prs.jsonl"
SEEN_CANON=""
for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
if [[ "$repo_name" == .* && "$repo_name" != ".github" ]] || [ ! -d "$repo_dir/.git" ]; then
continue
fi

gh pr list --repo "$ORG/$repo_name" \
# Core-repo allowlist + canonical remap + dedup of duplicate clone dirs.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi
case " $SEEN_CANON " in
*" $canon "*) continue ;;
esac
SEEN_CANON="$SEEN_CANON $canon"

gh pr list --repo "rossoctl/$canon" \
--author "app/dependabot" \
--state merged \
--json number,createdAt,mergedAt \
Expand Down Expand Up @@ -149,15 +160,26 @@ echo "--- Discovering scanner issues ---"
: > "$TMPDIR/issues.jsonl"
REPOS_CHECKED=0

SEEN_CANON=""
for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
if [[ "$repo_name" == .* && "$repo_name" != ".github" ]] || [ ! -d "$repo_dir/.git" ]; then
continue
fi

# Core-repo allowlist + canonical remap + dedup of duplicate clone dirs.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi
case " $SEEN_CANON " in
*" $canon "*) continue ;;
esac
SEEN_CANON="$SEEN_CANON $canon"

REPOS_CHECKED=$((REPOS_CHECKED + 1))
full_repo="$ORG/$repo_name"
full_repo="rossoctl/$canon"

issues_json=$(gh issue list --repo "$full_repo" \
--search "[dep-bump] in:title" \
Expand Down Expand Up @@ -514,14 +536,25 @@ echo "--- Computing metrics ---"

# Query recently merged Dependabot PRs (last 30 days) for TTM
: > "$TMPDIR/recent_merged.jsonl"
SEEN_CANON_TTM=""
for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
if [[ "$repo_name" == .* && "$repo_name" != ".github" ]] || [ ! -d "$repo_dir/.git" ]; then
continue
fi

gh pr list --repo "$ORG/$repo_name" \
# Core-repo allowlist + canonical remap + dedup of duplicate clone dirs.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi
case " $SEEN_CANON_TTM " in
*" $canon "*) continue ;;
esac
SEEN_CANON_TTM="$SEEN_CANON_TTM $canon"

gh pr list --repo "rossoctl/$canon" \
--author "app/dependabot" \
--state merged \
--json number,createdAt,mergedAt \
Expand Down
28 changes: 22 additions & 6 deletions scripts/dep-bump-scanner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ echo "--- Detecting ecosystems ---"
: > "$TMPDIR/ecosystems.jsonl"
REPOS_SCANNED=0

SEEN_CANON=""
for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
Expand All @@ -111,6 +112,17 @@ for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
continue
fi

# Restrict to core repos (allowlist) via the canonical name; dedup duplicate
# clone dirs so each canonical repo is processed once.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi
case " $SEEN_CANON " in
*" $canon "*) continue ;;
esac
SEEN_CANON="$SEEN_CANON $canon"

REPOS_SCANNED=$((REPOS_SCANNED + 1))
ecosystems=""

Expand Down Expand Up @@ -147,7 +159,8 @@ for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
# Strip trailing comma
ecosystems="${ecosystems%,}"

jq -nc --arg repo "$repo_name" --arg eco "$ecosystems" \
# Store the canonical repo name so Step 2 builds correct rossoctl/<name> refs.
jq -nc --arg repo "$canon" --arg eco "$ecosystems" \
'{repo: $repo, ecosystems: ($eco | split(",") | map(select(. != "")))}' \
>> "$TMPDIR/ecosystems.jsonl"
done
Expand All @@ -163,12 +176,14 @@ REPOS_WITH_DEPENDABOT=0
TOTAL_OPEN_PRS=0

while IFS= read -r eco_record; do
repo_name=$(echo "$eco_record" | jq -r '.repo')
full_repo="$ORG/$repo_name"
repo_name=$(echo "$eco_record" | jq -r '.repo') # canonical name from Step 1
full_repo="rossoctl/$repo_name"

echo " Checking $repo_name..."

# List open Dependabot PRs
# List open Dependabot PRs. Query the canonical repo directly -- gh pr list
# with --author silently returns empty across a rename redirect, which is why
# the pre-rename names reported zero Dependabot PRs.
prs_json=$(gh pr list --repo "$full_repo" \
--author "app/dependabot" \
--state open \
Expand Down Expand Up @@ -418,7 +433,8 @@ while IFS='|' read -r issue_repo issue_pr_number; do
category=$(echo "$record" | jq -r '.category')
overdue=$((age_days - sla_days))

full_repo="$ORG/$issue_repo"
# Record repos are canonical names (see Step 1); build canonical refs.
full_repo="rossoctl/$issue_repo"

# Deduplication
search_term="[dep-bump] Stale $severity bump: $package in $issue_repo"
Expand Down Expand Up @@ -497,7 +513,7 @@ ISSUES_CLOSED=0
while IFS='|' read -r fix_repo fix_pr_number; do
[ -z "$fix_repo" ] && continue

full_repo="$ORG/$fix_repo"
full_repo="rossoctl/$fix_repo"

# Find matching open issue by searching for the PR number in title/body
issue_number=$(gh issue list --repo "$full_repo" \
Expand Down
17 changes: 9 additions & 8 deletions scripts/extract-broken-links.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,18 @@ set -euo pipefail
# here so it can be unit-tested against synthetic lychee fixtures.
#
# Usage:
# extract-broken-links.sh <lychee-json> <repo-name> <repos-prefix>
# cat report.json | extract-broken-links.sh - <repo-name> <repos-prefix>
# extract-broken-links.sh <lychee-json> <repo> <repos-prefix>
# cat report.json | extract-broken-links.sh - <repo> <repos-prefix>
#
# Arguments:
# <lychee-json> Path to a lychee JSON report, or - to read from stdin.
# <repo-name> Bare repo name (e.g. "kagenti"); emitted as "kagenti/<name>".
# <repo> Full "owner/name" repo reference (e.g. "rossoctl/cortex");
# emitted verbatim in the "repo" field.
# <repos-prefix> Absolute path prefix stripped from lychee's file keys
# (e.g. "/home/claw/kagenti/kagenti/").
# (e.g. "/home/claw/kagenti/rossoctl/").
#
# Output (stdout): JSONL, zero or more objects of the form
# {"repo":"kagenti/foo","file":"docs/x.md","url":"https://...","status":"404","category":"external"}
# {"repo":"rossoctl/foo","file":"docs/x.md","url":"https://...","status":"404","category":"external"}
#
# Exit codes:
# 0 - success (may emit zero records)
Expand All @@ -38,7 +39,7 @@ if [ $# -lt 3 ]; then
fi

LYCHEE_INPUT="$1"
REPO_NAME="$2"
REPO_FULL="$2" # full "owner/name" reference, emitted verbatim
REPOS_PREFIX="$3"

if [ "$LYCHEE_INPUT" != "-" ] && [ ! -f "$LYCHEE_INPUT" ]; then
Expand All @@ -55,7 +56,7 @@ fi
# Normalize lychee status to enum tokens: numeric codes stay as-is,
# text statuses map to: timeout, dns, unreachable, error, unknown.
# Suppress URLs with unreachable-by-design hostnames (cluster-local, .local, RFC1918).
jq -r --arg repo "$REPO_NAME" --arg repos_dir "$REPOS_PREFIX" '
jq -r --arg repo "$REPO_FULL" --arg repos_dir "$REPOS_PREFIX" '
.error_map // {} | to_entries[] |
.key as $filepath |
.value[] |
Expand All @@ -79,7 +80,7 @@ jq -r --arg repo "$REPO_NAME" --arg repos_dir "$REPOS_PREFIX" '
end
) as $status |
{
repo: ("kagenti/" + $repo),
repo: $repo,
file: ($filepath | ltrimstr($repos_dir) | ltrimstr("./")),
url: .url,
status: $status,
Expand Down
14 changes: 13 additions & 1 deletion scripts/link-health-fixer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,26 @@ echo "=== Step 1: Gathering open scanner issues ==="
ISSUES_FILE="$TMPDIR/issues.jsonl"
: > "$ISSUES_FILE"

SEEN_CANON=""
for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
if [[ "$repo_name" == .* && "$repo_name" != ".github" ]] || [ ! -d "$repo_dir/.git" ]; then
continue
fi

full_repo="$ORG/$repo_name"
# Restrict to core repos (allowlist), mapping pre-rename dir names first;
# dedup so duplicate clone dirs for the same canonical repo run once.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi
case " $SEEN_CANON " in
*" $canon "*) continue ;;
esac
SEEN_CANON="$SEEN_CANON $canon"

full_repo="rossoctl/$canon"

issues_json=$(gh issue list --repo "$full_repo" \
--search "Broken link in:title" \
Expand Down
21 changes: 19 additions & 2 deletions scripts/link-health-scanner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ REPOS_FAILED=0
# Collect all broken links into a single JSONL file
: > "$TMPDIR/broken.jsonl"

# Track canonical repos already scanned this run, so duplicate clone dirs
# (e.g. a stale "kagenti" alongside "rossoctl") are not scanned twice.
SEEN_CANON=""

for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
[ -d "$repo_dir" ] || continue
repo_name=$(basename "$repo_dir")
Expand All @@ -72,7 +76,20 @@ for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
continue
fi

echo "Scanning $repo_name..."
# Restrict to core repos (allowlist), mapping pre-rename dir names to their
# canonical repo first. Non-core / archived clones are skipped.
canon=$(canonical_repo_for_dir "$repo_name")
if ! is_core_repo "$canon"; then
continue
fi

# Dedup: skip if another clone dir already covered this canonical repo.
case " $SEEN_CANON " in
*" $canon "*) echo "Skipping $repo_name (already scanned as $canon)"; continue ;;
esac
SEEN_CANON="$SEEN_CANON $canon"

echo "Scanning $repo_name (as rossoctl/$canon)..."

LYCHEE_OUTPUT="$TMPDIR/lychee_${repo_name}.json"

Expand Down Expand Up @@ -114,7 +131,7 @@ for repo_dir in "$REPOS_DIR"/*/ "$REPOS_DIR"/.github/; do
# normalization logic lives in extract-broken-links.sh so it can be unit-tested
# (see tests/test-extract-broken-links.sh).
"$SCRIPT_DIR/extract-broken-links.sh" \
"$LYCHEE_OUTPUT" "$repo_name" "$REPOS_DIR/$repo_name/" \
"$LYCHEE_OUTPUT" "rossoctl/$canon" "$REPOS_DIR/$repo_name/" \
>> "$TMPDIR/broken.jsonl" 2>/dev/null || true

echo " Links: $repo_total, Errors: $repo_errors"
Expand Down
23 changes: 23 additions & 0 deletions scripts/program-lib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -841,12 +841,35 @@ core_repo_names() {
get_core_repos | sed 's|^[^/]*/||'
}

# Return 0 if the given bare repo name is in the core allowlist, else 1.
#
# Intended for filtering local clone iteration: pass the canonical name
# (see canonical_repo_for_dir) so pre-rename clone dirs are matched correctly.
# Uses an exact, whole-line match (grep -Fx) to avoid substring false positives.
#
# Usage:
# canon=$(canonical_repo_for_dir "$repo_name")
# is_core_repo "$canon" || continue
# Args:
# $1 - bare repo name (no owner prefix)
is_core_repo() {

@cwiklik cwiklik Jul 28, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (non-blocking): the allowlist + canonical-remap + dedup block is now copy-pasted ~8 times across the five scanners (3× in dep-bump-fixer alone), which is real drift risk for security-sensitive automation. Consider a small program-lib helper — e.g. canon=$(repo_filter "$dir") || continue that performs the canonical_repo_for_dir remap + is_core_repo gate and echoes the canonical name, leaving only the SEEN_CANON dedup inline — so the allowlist policy lives in exactly one place.

@rubambiza rubambiza Jul 28, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great suggestion. This reinforces what I had already noticed. That is, program-lib.sh was growing too long for easy maintainability, and I have an open issue as part of the epic to refactor it into a smaller set of subprograms like you suggested. I will add this suggestion to the context for the planning.

@rubambiza rubambiza Jul 28, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See Design Input section in #35

local name="$1"
core_repo_names | grep -qxF "$name"
}

# Map a local clone directory basename to its canonical bare repo name.
#
# Clone dirs may still use pre-rename names; this encapsulates the rename
# remap table in one place so every script agrees. Unknown names pass through
# unchanged (identity), so non-remapped repos need no special handling.
#
# TRANSITIONAL: the non-identity entries below are a temporary bridge for the
# kagenti->rossoctl rename while stale-named clone dirs still exist on disk.
# Once clone dirs are renamed to canonical names, this function becomes pure
# identity and the entries should be deleted. See rossoctl/automation#37.
# It is a lookup table, not a rename detector -- do not treat it as protection
# against future renames.
#
# Returns: the bare repo name only (e.g. "rossoctl"), NOT an owner/name pair.
# Prepend the owner to build a full API reference, e.g. "rossoctl/$canon".
#
Expand Down
11 changes: 11 additions & 0 deletions tests/test-core-repos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@ if CORE_REPOS_FILE="$TEST_TMPDIR/empty.txt" get_core_repos >/dev/null 2>&1; then
echo "FAIL get_core_repos should error on empty allowlist"; fail=1
fi

# --- is_core_repo: membership test against the allowlist (exact, whole-line) ---
CORE_REPOS_FILE="$TEST_TMPDIR/good.txt" is_core_repo "rossoctl" \
|| { echo "FAIL is_core_repo: rossoctl should be in allowlist"; fail=1; }
if CORE_REPOS_FILE="$TEST_TMPDIR/good.txt" is_core_repo "not-a-repo"; then
echo "FAIL is_core_repo: not-a-repo should NOT match"; fail=1
fi
# Guard against substring false positives (rosso is a prefix of rossoctl).
if CORE_REPOS_FILE="$TEST_TMPDIR/good.txt" is_core_repo "rosso"; then
echo "FAIL is_core_repo: partial 'rosso' must not match 'rossoctl'"; fail=1
fi

# --- canonical_repo_for_dir: remaps the two renamed dirs, identity otherwise ---
[ "$(canonical_repo_for_dir kagenti)" = "rossoctl" ] \
|| { echo "FAIL canonical: kagenti -> rossoctl"; fail=1; }
Expand Down
7 changes: 4 additions & 3 deletions tests/test-extract-broken-links.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ EXTRACTOR="$SCRIPT_DIR/../scripts/extract-broken-links.sh"
TEST_TMPDIR=$(mktemp -d "/tmp/test-extract-broken-XXXXXX")
trap 'rm -rf "$TEST_TMPDIR"' EXIT

REPO="kagenti-extensions"
REPOS_PREFIX="/home/claw/kagenti/kagenti-extensions/"
# Arg 2 is now a full "owner/name" reference, emitted verbatim in .repo.
REPO="rossoctl/cortex"
REPOS_PREFIX="/home/claw/kagenti/cortex/"

PASS=0
FAIL=0
Expand Down Expand Up @@ -101,7 +102,7 @@ write_fixture "$TEST_TMPDIR/ext.json" \
run_test "external record count" "$TEST_TMPDIR/ext.json" 'length' '1'
run_test "external status normalized" "$TEST_TMPDIR/ext.json" '.[0].status' '404'
run_test "external category" "$TEST_TMPDIR/ext.json" '.[0].category' 'external'
run_test "external repo prefixed" "$TEST_TMPDIR/ext.json" '.[0].repo' 'kagenti/kagenti-extensions'
run_test "external repo emitted verbatim" "$TEST_TMPDIR/ext.json" '.[0].repo' 'rossoctl/cortex'
run_test "external file prefix stripped" "$TEST_TMPDIR/ext.json" '.[0].file' 'docs/d.md'
run_test "external url preserved" "$TEST_TMPDIR/ext.json" '.[0].url' 'https://example.invalid/gone'

Expand Down