diff --git a/ops/gmpctl/.gmpctl.default.yaml b/ops/gmpctl/.gmpctl.default.yaml index b44eb2fef8..5201f042df 100644 --- a/ops/gmpctl/.gmpctl.default.yaml +++ b/ops/gmpctl/.gmpctl.default.yaml @@ -13,3 +13,10 @@ # limitations under the License. dir: "./data" + +# Go modules to ignore during dependency upgrades (e.g., vulnfix). +vuln_ignored_modules: + # Until unfork, upgrading prometheus is non-trivial and its vulnerabilities + # are typically not related to us. Skip it in vulnerability fixes, and produce warning. + - "github.com/prometheus/prometheus" + diff --git a/ops/gmpctl/README.md b/ops/gmpctl/README.md index 42da662e7f..f4049974eb 100644 --- a/ops/gmpctl/README.md +++ b/ops/gmpctl/README.md @@ -17,10 +17,26 @@ It's a starting point for smaller or bigger automation on OSS side (e.g. releasi * `gsed` (MacOS: `brew install gsed`) * `gcloud` (https://docs.cloud.google.com/sdk/docs/install-sdk) (and `gcloud auth login`) -3. You can configure different work directory for gmpctl via `-c` flag. By default, `gmpctl` does the work in `ops/gmpctl/.data` +3. You can configure different work directory and settings for gmpctl via `-c` flag (default configuration file: `.gmpctl.default.yaml`). Enjoy! +## Configuration + +`gmpctl` reads configuration from a YAML file specified by the `-c` flag (defaults to `.gmpctl.default.yaml`). + +Available options: + +* `dir`: Base directory for gmpctl work, project clones, and git worktrees (default: `./data`). +* `vuln_ignored_modules`: List of Go module paths to ignore during module upgrades (e.g. `vulnfix`). + Example: + + ```yaml + dir: "./data" + vuln_ignored_modules: + - "github.com/prometheus/prometheus" + ``` + ## Usage Generally `gmpctl` does not need flags for general usage. It interactively asks you for diff --git a/ops/gmpctl/cmd_vulnfix.go b/ops/gmpctl/cmd_vulnfix.go index b6051b4962..6f7c49bdcc 100644 --- a/ops/gmpctl/cmd_vulnfix.go +++ b/ops/gmpctl/cmd_vulnfix.go @@ -19,6 +19,7 @@ import ( "flag" "fmt" "os" + "strings" ) var ( @@ -78,6 +79,7 @@ func vulnfix() error { fmt.Sprintf("DIR=%v", dir), fmt.Sprintf("BRANCH=%v", branch), fmt.Sprintf("PROJECT=%v", proj.Name), + fmt.Sprintf("VULN_IGNORED_MODULES=%v", strings.Join(cfg.VulnIgnoredModules, ",")), } if *vulnfixSyncDockerfilesFrom { opts = append(opts, "SYNC_DOCKERFILES_FROM=true") diff --git a/ops/gmpctl/go.mod b/ops/gmpctl/go.mod index fd911410e6..f4d6d91c80 100644 --- a/ops/gmpctl/go.mod +++ b/ops/gmpctl/go.mod @@ -56,9 +56,9 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/oauth2 v0.33.0 // indirect - golang.org/x/sync v0.18.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.29.0 // indirect + golang.org/x/text v0.40.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/ops/gmpctl/go.sum b/ops/gmpctl/go.sum index 6d778d881b..f64cea62b8 100644 --- a/ops/gmpctl/go.sum +++ b/ops/gmpctl/go.sum @@ -126,21 +126,21 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/ops/gmpctl/lib.sh b/ops/gmpctl/lib.sh index 7d5801925c..dc6e520aee 100755 --- a/ops/gmpctl/lib.sh +++ b/ops/gmpctl/lib.sh @@ -111,7 +111,8 @@ release-lib::vulnlist() { go run "./..." \ -go-version="${go_version}" \ -only-fixed \ - -dir="${dir}" | tee "${vuln_file}" + -dir="${dir}" \ + -vuln-ignore-modules="${VULN_IGNORED_MODULES:-}" | tee "${vuln_file}" if [[ -z $(cat "${vuln_file}") ]]; then # Print this, otherwise error on the above might keep this file mistakenly empty. echo "no vulnerabilities" >"${vuln_file}" @@ -473,7 +474,6 @@ release-lib::manifests_regen() { echo "🔄 Regenerating manifests..." YQ="$(command -v yq)" HELM="$(command -v helm)" ADDLICENSE="$(command -v addlicense)" bash "${dir}/hack/presubmit.sh" manifests - if [[ -f "${dir}/.bingo/variables.env.bak" ]]; then mv "${dir}/.bingo/variables.env.bak" "${dir}/.bingo/variables.env" trap - EXIT diff --git a/ops/gmpctl/main.go b/ops/gmpctl/main.go index 602e66823f..c2a73cc4a4 100644 --- a/ops/gmpctl/main.go +++ b/ops/gmpctl/main.go @@ -36,6 +36,8 @@ var ( type Config struct { // Directory for the gmpctl work, notably for project clones and git worktrees. Directory string `yaml:"dir"` + // VulnIgnoredModules contains Go module paths to ignore during upgrades (e.g. vulnfix). + VulnIgnoredModules []string `yaml:"vuln_ignored_modules"` } func loadConfig() (ret *Config, _ error) { diff --git a/ops/gmpctl/vulnfix.sh b/ops/gmpctl/vulnfix.sh index caf90053f7..5d56869867 100644 --- a/ops/gmpctl/vulnfix.sh +++ b/ops/gmpctl/vulnfix.sh @@ -100,23 +100,27 @@ echo "*" >>"${DIR}/.gmpctl/.gitignore" vuln_file="${DIR}/.gmpctl/vulnlist.txt" pushd "${DIR}" -release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" +max_attempts=5 +for ((i=1; i<=max_attempts; i++)); do + release-lib::vulnlist "${DIR}" "${vuln_file}" + if [[ "no vulnerabilities" == $(cat "${vuln_file}") ]]; then + echo "✅ No more fixable Go vulnerabilities found!" + break + fi -if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - # Attempt to update + go mod tidy. + echo "🔄 Iteration $i/$max_attempts: Applying dependency updates..." release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" git add go.mod go.sum - if [ -d "${DIR}/vendor" ]; then go mod vendor git add --all fi +done - # Check if that helped. - echo "⚠️ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." +if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then release-lib::vulnlist "${DIR}" "${vuln_file}" - if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually (select not reusing the ./vulnlist.txt file) and rerun." - exit 1 - fi +fi +if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually." + exit 1 fi diff --git a/ops/gmpctl/vulnupdatelist/main.go b/ops/gmpctl/vulnupdatelist/main.go index 30b216f5e7..f02eee33c0 100644 --- a/ops/gmpctl/vulnupdatelist/main.go +++ b/ops/gmpctl/vulnupdatelist/main.go @@ -34,14 +34,16 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/Masterminds/semver/v3" ) var ( - goVersion = flag.String("go-version", "", "Go version to test vulnerabilities in (stdlib). Otherwise the `go env GOVERSION` is used") - dir = flag.String("dir", ".", "Where to run the script from") - onlyFixed = flag.Bool("only-fixed", false, "Don't print vulnerable modules without fixed version; note: fixed version often means sometimes that a new major version contains a fix.") + goVersion = flag.String("go-version", "", "Go version to test vulnerabilities in (stdlib). Otherwise the `go env GOVERSION` is used") + dir = flag.String("dir", ".", "Where to run the script from") + onlyFixed = flag.Bool("only-fixed", false, "Don't print vulnerable modules without fixed version; note: fixed version often means sometimes that a new major version contains a fix.") + vulnIgnoreModules = flag.String("vuln-ignore-modules", "", "Comma-separated list of Go module paths to ignore for upgrades due to known issues.") ) // UpdateList presents the minimum version to upgrade to solve all CVEs with @@ -89,8 +91,18 @@ func main() { os.Exit(0) } + ignoredModules := make(map[string]struct{}) + if *vulnIgnoreModules != "" { + for _, m := range strings.Split(*vulnIgnoreModules, ",") { + m = strings.TrimSpace(m) + if m != "" { + ignoredModules[m] = struct{}{} + } + } + } + slog.Info("Parsing vulnerabilities and finding updates...") - updates, err := compileUpdateList(bytes.NewReader(vulnJSON), *onlyFixed) + updates, err := compileUpdateList(bytes.NewReader(vulnJSON), *onlyFixed, ignoredModules) if err != nil { log.Fatalf("Error parsing govulncheck output: %v", err) } diff --git a/ops/gmpctl/vulnupdatelist/vuln.go b/ops/gmpctl/vulnupdatelist/vuln.go index f4176602ab..4c21188078 100644 --- a/ops/gmpctl/vulnupdatelist/vuln.go +++ b/ops/gmpctl/vulnupdatelist/vuln.go @@ -82,7 +82,7 @@ type FindingTrace struct { // compileUpdateList decodes the JSON stream from govulncheck and extracts // a list of modules that need to be updated to a fixed version. -func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) { +func compileUpdateList(jsonData io.Reader, onlyFixed bool, ignoredModules map[string]struct{}) ([]UpdateList, error) { updates := make(map[string]UpdateList) osvs := make(map[string]*OSV) decoder := json.NewDecoder(jsonData) @@ -109,10 +109,8 @@ func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) // We assume OSVs are printed first. osv := osvs[v.Finding.OSVID] cveID := v.Finding.OSVID - allCVEs := v.Finding.OSVID if osv != nil { cveID = getCVEID(*osv) - allCVEs = osv.CVEs() } else { slog.Error("Malformed govulncheck input; a finding without an OSV entry.", "finding.osv", v.Finding.OSVID) } @@ -132,14 +130,8 @@ func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) } } - if onlyFixed && fixVersion == nil { - slog.Warn("IMPORTANT: Found Go vulnerability without a fixed version. Ignoring this module, given the -only-fixed flag...", "mod", module, "cve", cveID) - continue - } - up, ok := updates[module] if !ok { - slog.Info("Found Go vulnerability with a fix; queuing...", "mod", module, "CVEs", allCVEs) updates[module] = UpdateList{ CVEID: cveID, Module: module, @@ -149,8 +141,6 @@ func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) continue } - // Check if there are more CVE IDs corresponding to the vulnerability, which can give more context. - slog.Debug("Found Go vulnerability with a fix, the module was already queued; resolving version...", "mod", module, "CVEs", allCVEs) up.AdditionalCVEs++ if fixVersion != nil { if up.FixedVersion == nil || fixVersion.GreaterThan(up.FixedVersion) { @@ -160,10 +150,26 @@ func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) updates[module] = up } - updateList := slices.Collect(maps.Values(updates)) - sort.Slice(updateList, func(i, j int) bool { - return updateList[i].Module < updateList[j].Module + allUpdates := slices.Collect(maps.Values(updates)) + sort.Slice(allUpdates, func(i, j int) bool { + return allUpdates[i].Module < allUpdates[j].Module }) + + var updateList []UpdateList + for _, up := range allUpdates { + if _, ignored := ignoredModules[up.Module]; ignored { + slog.Warn("IMPORTANT: Found Go vulnerability in an ignored module; skipping upgrade...", "module", up.Module, "cve", up.CVEID) + continue + } + + if onlyFixed && up.FixedVersion == nil { + slog.Warn("IMPORTANT: Found Go vulnerability without a fixed version. Ignoring this module, given the -only-fixed flag...", "mod", up.Module, "cve", up.CVEID) + continue + } + + slog.Info("Found Go vulnerability with a fix; queuing...", "mod", up.Module, "fixedVersion", up.FixedVersion, "cve", up.CVEID) + updateList = append(updateList, up) + } return updateList, nil } diff --git a/ops/gmpctl/vulnupdatelist/vuln_test.go b/ops/gmpctl/vulnupdatelist/vuln_test.go new file mode 100644 index 0000000000..21fedbdd70 --- /dev/null +++ b/ops/gmpctl/vulnupdatelist/vuln_test.go @@ -0,0 +1,63 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompileUpdateList_IgnoredModules(t *testing.T) { + mockJSON := ` +{"osv":{"id":"GO-2024-0001","summary":"Test vuln 1"}} +{"finding":{"osv":"GO-2024-0001","fixed_version":"1.2.3","trace":[{"module":"github.com/prometheus/prometheus","version":"1.0.0"}]}} +{"osv":{"id":"GO-2024-0002","summary":"Test vuln 2"}} +{"finding":{"osv":"GO-2024-0002","fixed_version":"0.1.5","trace":[{"module":"golang.org/x/net","version":"0.1.0"}]}} +` + + t.Run("without ignored modules", func(t *testing.T) { + updates, err := compileUpdateList(strings.NewReader(mockJSON), false, nil) + require.NoError(t, err) + require.Len(t, updates, 2) + + require.Equal(t, "github.com/prometheus/prometheus", updates[0].Module) + require.Equal(t, "GO-2024-0001", updates[0].CVEID) + require.Equal(t, "1.0.0", updates[0].Version) + require.NotNil(t, updates[0].FixedVersion) + require.Equal(t, "1.2.3", updates[0].FixedVersion.String()) + + require.Equal(t, "golang.org/x/net", updates[1].Module) + require.Equal(t, "GO-2024-0002", updates[1].CVEID) + require.Equal(t, "0.1.0", updates[1].Version) + require.NotNil(t, updates[1].FixedVersion) + require.Equal(t, "0.1.5", updates[1].FixedVersion.String()) + }) + + t.Run("with ignored module", func(t *testing.T) { + ignored := map[string]struct{}{ + "github.com/prometheus/prometheus": {}, + } + updates, err := compileUpdateList(strings.NewReader(mockJSON), false, ignored) + require.NoError(t, err) + require.Len(t, updates, 1) + require.Equal(t, "golang.org/x/net", updates[0].Module) + require.Equal(t, "GO-2024-0002", updates[0].CVEID) + require.Equal(t, "0.1.0", updates[0].Version) + require.NotNil(t, updates[0].FixedVersion) + require.Equal(t, "0.1.5", updates[0].FixedVersion.String()) + }) +}