From 71cb7fc1759556f960253e02ff216ed5d5212f63 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 17:15:57 -0700 Subject: [PATCH 01/34] feat(gpu): pre-install NVIDIA driver in Ubuntu base image for air-gapped GPU Operator Add scripts/install-nvidia-drivers.sh to bake the NVIDIA data-center driver and its DKMS kernel modules into the Ubuntu base image, so GPU nodes can run the NVIDIA GPU Operator with driver.enabled=false in fully air-gapped environments (no host-side network, no operator driver container). The script derives the target kernel from /lib/modules (not the builder's uname -r), installs ABI-exact headers via install-kernel-headers.sh, forces DKMS build/install/depmod against that kernel, and verifies nvidia*.ko landed in the image kernel tree. It also blacklists nouveau, autoloads the nvidia stack, enables nvidia-persistenced, and rebuilds the initrd. Wire it into the base-image target (Ubuntu/amd64, after the kernel is finalized) behind INSTALL_NVIDIA_GPU_DRIVERS, with tunable branch/type/repo/fabricmanager/ toolkit args. Document in .arg.template and docs/nvidia-gpu-airgapped.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- .arg.template | 12 ++ Earthfile | 28 +++ docs/nvidia-gpu-airgapped.md | 166 +++++++++++++++ scripts/install-nvidia-drivers.sh | 328 ++++++++++++++++++++++++++++++ 4 files changed, 534 insertions(+) create mode 100644 docs/nvidia-gpu-airgapped.md create mode 100755 scripts/install-nvidia-drivers.sh diff --git a/.arg.template b/.arg.template index a27d42bd..4460591d 100644 --- a/.arg.template +++ b/.arg.template @@ -31,3 +31,15 @@ FORCE_INTERACTIVE_INSTALL=false # AUTO_ENROLL_SECUREBOOT_KEYS=false # Set to true to automatically enroll certificates on devices in Setup Mode, useful for flashing devices without user interaction # DRBD_VERSION=9.2.13 # This variable is required for Piraeus pack for drbd module installtion. + +# NVIDIA GPU driver pre-install (for running the NVIDIA GPU Operator in air-gapped +# environments with driver.enabled=false). Bakes the datacenter driver + DKMS +# kernel modules into the Ubuntu base image so GPU nodes need no host-side network. +# See scripts/install-nvidia-drivers.sh for details. +# INSTALL_NVIDIA_GPU_DRIVERS=true +# NVIDIA_DRIVER_BRANCH=570 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') +# NVIDIA_DRIVER_TYPE=proprietary # proprietary | open (open GPU kernel modules; Turing+ only) +# NVIDIA_USE_CUDA_REPO=true # Add NVIDIA CUDA network repo at build time (has every -server branch) +# NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems +# NVIDIA_INSTALL_CONTAINER_TOOLKIT=false # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false) +# NVIDIA_REBUILD_INITRD=true # Rebuild initrd so the nouveau blacklist applies at early boot diff --git a/Earthfile b/Earthfile index 493d6100..ed34f655 100644 --- a/Earthfile +++ b/Earthfile @@ -63,6 +63,17 @@ ARG no_proxy=${NO_PROXY} ARG UPDATE_KERNEL=false +# NVIDIA GPU driver pre-install (for air-gapped GPU Operator with driver.enabled=false). +# When true, the NVIDIA data-center driver + DKMS kernel modules are baked into the +# Ubuntu base image so GPU nodes need no host-side network at boot. +ARG INSTALL_NVIDIA_GPU_DRIVERS=false +ARG NVIDIA_DRIVER_BRANCH=570 +ARG NVIDIA_DRIVER_TYPE=proprietary +ARG NVIDIA_USE_CUDA_REPO=true +ARG NVIDIA_INSTALL_FABRICMANAGER=false +ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false +ARG NVIDIA_REBUILD_INITRD=true + IF [ "$FIPS_ENABLED" = "true" ] && [ "$UPDATE_KERNEL" = "true" ] RUN echo "ERROR: UPDATE_KERNEL and FIPS_ENABLED are mutually exclusive. Cannot set both to true." >&2 && \ exit 1 @@ -797,6 +808,23 @@ base-image: fi END + # NVIDIA GPU driver + DKMS kernel modules, built against the now-finalized + # image kernel. Runs here (not in the Dockerfile) so the kernel is settled + # first. Reuses install-kernel-headers.sh for ABI-exact headers. + IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-nvidia-drivers.sh /tmp/install-nvidia-drivers.sh + RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-nvidia-drivers.sh && \ + NVIDIA_DRIVER_BRANCH="$NVIDIA_DRIVER_BRANCH" \ + NVIDIA_DRIVER_TYPE="$NVIDIA_DRIVER_TYPE" \ + NVIDIA_USE_CUDA_REPO="$NVIDIA_USE_CUDA_REPO" \ + NVIDIA_INSTALL_FABRICMANAGER="$NVIDIA_INSTALL_FABRICMANAGER" \ + NVIDIA_INSTALL_CONTAINER_TOOLKIT="$NVIDIA_INSTALL_CONTAINER_TOOLKIT" \ + NVIDIA_REBUILD_INITRD="$NVIDIA_REBUILD_INITRD" \ + /tmp/install-nvidia-drivers.sh && \ + rm -f /tmp/install-nvidia-drivers.sh /tmp/install-kernel-headers.sh + END + IF [ "$CIS_HARDENING" = "true" ] COPY cis-harden/harden.sh /tmp/harden.sh RUN /tmp/harden.sh && rm /tmp/harden.sh diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md new file mode 100644 index 00000000..1ba2956a --- /dev/null +++ b/docs/nvidia-gpu-airgapped.md @@ -0,0 +1,166 @@ +# Pre-installing the NVIDIA GPU driver for air-gapped GPU Operator + +This guide explains how to bake the NVIDIA data-center GPU driver and its +kernel modules **into a CanvOS Ubuntu base image**, so that GPU nodes can run +the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-air-gapped.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator's driver container**. + +- Script: [`scripts/install-nvidia-drivers.sh`](../scripts/install-nvidia-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_NVIDIA_GPU_DRIVERS=true`. + +--- + +## Why do this (the split of responsibilities) + +The GPU Operator normally deploys the NVIDIA driver as a **driver container** that +downloads and compiles the driver at runtime. That requires network access on the +node and a matching kernel-headers source — neither of which exists on an +air-gapped, immutable (Kairos) edge host. + +The supported alternative is the **pre-installed driver** model: + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| NVIDIA kernel driver + modules (`nvidia`, `nvidia_uvm`, `nvidia_modeset`, `nvidia_drm`) | **In the OS image** | **This script (build time)** | +| `nvidia-smi` / driver user-space | **In the OS image** | **This script (build time)** | +| nvidia-container-toolkit / runtime class | Container image | GPU Operator (from your content bundle) | +| device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, validator | Container images | GPU Operator (from your content bundle) | + +So: **the OS carries only the driver + kernel modules**; everything else is a +container image you mirror into your Palette content bundle. At boot the node has a +working driver with zero connectivity, and once your bundled operator images are +present the GPU cluster comes up with no external pulls. + +At Helm-install time you **must** tell the operator the driver is pre-installed: + +``` +--set driver.enabled=false +``` + +If you also opt in to pre-installing the container toolkit on the host +(`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true`, off by default), additionally set: + +``` +--set toolkit.enabled=false +``` + +--- + +## The key build-time problem this solves + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, **not** +the kernel baked into the image. If DKMS builds "for the running kernel", you get +modules for the wrong ABI (or the build fails). The script therefore: + +1. derives the **target kernel** from `/lib/modules/*` (the kernel that will boot), +2. installs **ABI-exact kernel headers** for it (reusing + [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh), which falls + back to `snapshot.ubuntu.com` when Ubuntu rotates the ABI out of the live mirror), +3. forces **DKMS build + install + `depmod`** against that target kernel, and +4. **verifies** the resulting `nvidia*.ko` modules actually landed under + `/lib/modules//` — failing the build loudly if they did not. + +It runs in the `base-image` target **after** the kernel is finalized +(hold/upgrade/purge/dracut), so modules are always built against the settled kernel. + +> **Connectivity note:** the build runs where the builder has internet and bakes +> everything into the image. The resulting image needs no network at boot. + +--- + +## Quick start + +1. Edit `.arg` (copied from `.arg.template`) and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22.04 + ARCH=amd64 + + INSTALL_NVIDIA_GPU_DRIVERS=true + NVIDIA_DRIVER_BRANCH=570 # verify the branch exists (see below) + NVIDIA_DRIVER_TYPE=proprietary # or: open (Turing+ only) + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line without touching `.arg`: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_NVIDIA_GPU_DRIVERS=true \ + --NVIDIA_DRIVER_BRANCH=570 \ + --NVIDIA_DRIVER_TYPE=proprietary + ``` + +3. Mirror the GPU Operator container images into your Palette content bundle + (per the NVIDIA air-gapped guide), and install the operator with + `driver.enabled=false`. + +--- + +## Configuration reference + +All variables are optional and have defaults. Set them in `.arg` or pass as +`--VAR=value` on the `earthly.sh` command line. + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | +| `NVIDIA_DRIVER_BRANCH` | `570` | Driver branch to install (e.g. `550`, `570`, `580`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | +| `NVIDIA_DRIVER_TYPE` | `proprietary` | `proprietary` or `open`. `open` uses the NVIDIA open GPU kernel modules (Turing architecture and newer only). | +| `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `false` | Set `true` for NVSwitch / HGX systems (installs and enables `nvidia-fabricmanager`). | +| `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | +| `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | + +### Choosing a driver branch + +Only certain branches publish the headless `-server` packages. Inside the base +image (or any Ubuntu 22.04 box with the CUDA repo added) you can list them: + +```sh +apt-cache search 'nvidia-headless-.*-server' +``` + +Pick a branch supported by both your GPU generation and the CUDA/toolkit versions +of the operator images you're bundling. + +--- + +## What the script configures on the host + +- `/etc/modprobe.d/blacklist-nouveau.conf` — blacklists the `nouveau` driver. +- `/etc/modules-load.d/nvidia.conf` — autoloads `nvidia`, `nvidia_uvm`, + `nvidia_modeset`, `nvidia_drm` at boot. +- `/etc/modprobe.d/nvidia.conf` — `NVreg_PreserveVideoMemoryAllocations=1`. +- Enables `nvidia-persistenced.service` (recommended for data-center GPUs). +- Runs `depmod -a ` and rebuilds the initrd for the target kernel. + +Verify on a booted node: + +```sh +nvidia-smi +lsmod | grep nvidia +``` + +--- + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path.** When `IS_UKI=true`, DKMS + modules are unsigned and will not load under Secure Boot; that requires MOK + signing, which this script does **not** implement. Use the standard (non-UKI) + Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** The script targets apt-based Ubuntu images on + `x86_64` (with a best-effort `sbsa` path for arm64). Non-Ubuntu distributions + are out of scope. +- **Branch/version alignment is yours to own.** Make sure `NVIDIA_DRIVER_BRANCH` + matches the GPU hardware and the CUDA/toolkit versions expected by the operator + images in your bundle. diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh new file mode 100755 index 00000000..28e1844b --- /dev/null +++ b/scripts/install-nvidia-drivers.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# +# install-nvidia-drivers.sh +# +# Pre-install the NVIDIA data-center GPU driver and build its kernel modules +# INTO a CanvOS / Kairos Ubuntu base image, so that a node booted from the +# image can run the NVIDIA GPU Operator in a fully air-gapped environment +# WITHOUT any host-side network access and WITHOUT the operator's driver +# container. +# +# WHAT THIS COVERS (OS side only) +# ------------------------------- +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers that match the kernel shipped in the image +# (delegated to scripts/install-kernel-headers.sh) +# * the NVIDIA driver user-space + `nvidia-smi` (nvidia-utils-*-server) +# * the NVIDIA kernel modules (nvidia, nvidia-uvm, nvidia-modeset, +# nvidia-drm, nvidia-peermem) built with DKMS against the IMAGE kernel +# * nouveau blacklist + nvidia module autoload + nvidia-persistenced +# * (optional) nvidia-fabricmanager for HGX / NVSwitch systems +# +# WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, +# deployed by the GPU Operator itself): +# * nvidia-container-toolkit / runtime class +# * k8s-device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, ... +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enabled=false +# (and, if you also pre-install the toolkit below, --set toolkit.enabled=false) +# +# WHY THE DKMS DANCE +# ------------------ +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. If we let apt/DKMS build "for the running kernel" the +# modules would target the wrong ABI (or fail). We therefore derive the target +# kernel from /lib/modules (the kernel that will actually boot) and force every +# DKMS build + module install + depmod against THAT kernel. +# +# CONNECTIVITY +# ------------ +# This script runs at BUILD time, where the builder has internet. It bakes +# everything into the image. The resulting image needs no network at boot. +# +# TUNABLES (environment variables; all optional) +# NVIDIA_DRIVER_BRANCH Driver branch to install (e.g. 550, 570, 580). +# Default: 570 (a data-center production branch) +# NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: proprietary +# (open = open GPU kernel modules; Turing+ only) +# NVIDIA_USE_CUDA_REPO "true" to add developer.download.nvidia.com CUDA +# repo (recommended, has every -server branch). +# "false" to use only Ubuntu's own repos. +# Default: true +# NVIDIA_INSTALL_FABRICMANAGER "true" for NVSwitch/HGX boxes. Default: false +# NVIDIA_INSTALL_CONTAINER_TOOLKIT "true" to ALSO pre-install +# nvidia-container-toolkit on the host (then set +# toolkit.enabled=false in the operator). +# Default: false (operator ships it) +# NVIDIA_REBUILD_INITRD "true" to rebuild the initrd so the nouveau +# blacklist takes effect in early boot. +# Default: true +# +set -u + +log() { echo "[install-nvidia-drivers] $*"; } +warn() { echo "[install-nvidia-drivers] WARNING: $*" >&2; } +die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-570}" +NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-proprietary}" +NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" +NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-false}" +NVIDIA_INSTALL_CONTAINER_TOOLKIT="${NVIDIA_INSTALL_CONTAINER_TOOLKIT:-false}" +NVIDIA_REBUILD_INITRD="${NVIDIA_REBUILD_INITRD:-true}" + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "Driver branch: ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE})" + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev pkg-config \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers matching the image kernel +# Reuse the repo's ABI-exact / snapshot-aware header installer if present. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + # Find the header tree that matches our kernel and symlink it. + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. NVIDIA package repo (CUDA network repo — has every *-server branch) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_USE_CUDA_REPO}" = "true" ]; then + # Derive the CUDA repo "distro" tag from the image (e.g. 22.04 -> ubuntu2204) + osid=""; osver="" + if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + osid="${ID:-ubuntu}" + osver="${VERSION_ID:-22.04}" + fi + distro="${osid}$(echo "${osver}" | tr -d '.')" # ubuntu2204, ubuntu2004, ... + case "$(uname -m)" in + x86_64) cudaarch="x86_64" ;; + aarch64) cudaarch="sbsa" ;; + *) cudaarch="x86_64" ;; + esac + repo_base="https://developer.download.nvidia.com/compute/cuda/repos/${distro}/${cudaarch}" + log "Adding NVIDIA CUDA repo: ${repo_base}" + if wget -qO /tmp/cuda-keyring.deb "${repo_base}/cuda-keyring_1.1-1_all.deb"; then + dpkg -i /tmp/cuda-keyring.deb || warn "cuda-keyring install failed." + rm -f /tmp/cuda-keyring.deb + apt-get update || warn "apt-get update after adding CUDA repo failed." + else + warn "could not download cuda-keyring; falling back to Ubuntu repos." + fi +fi + +# --------------------------------------------------------------------------- +# 5. Choose driver packages +# Headless server packages (no Xorg / GUI). nvidia-utils gives nvidia-smi. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server-open" +else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server" +fi +UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}-server" + +log "Installing NVIDIA driver packages: ${HEADLESS_PKG} ${UTILS_PKG}" +if ! apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}"; then + warn "'${HEADLESS_PKG}' not available; retrying with generic (non-server) branch." + if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-open" + else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}" + fi + UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}" + apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}" \ + || die "failed to install NVIDIA driver packages for branch ${NVIDIA_DRIVER_BRANCH}. \ +Check available branches with: apt-cache search 'nvidia-headless-.*-server'" +fi + +# --------------------------------------------------------------------------- +# 6. Build the DKMS modules against the IMAGE kernel (not the build host) +# --------------------------------------------------------------------------- +# The apt postinst runs `dkms autoinstall`, which only builds for kernels that +# have headers present -- i.e. our target kernel, since the build host kernel's +# headers are absent in the image. We still force it explicitly to be safe. +if command -v dkms >/dev/null 2>&1; then + log "Building NVIDIA DKMS modules for kernel ${KVER} ..." + # Explicitly (re)build every registered nvidia dkms module for the target. + # `dkms status` output differs across versions: + # dkms 2.x: "nvidia, 570.86.15, 6.14.0-36-generic, x86_64: installed" + # dkms 3.x: "nvidia/570.86.15, 6.14.0-36-generic, x86_64: installed" + # Extract the module name (up to the first , / or :) and the first + # version-looking token, which works for both formats. + dkms status 2>/dev/null | grep -i nvidia | while read -r line; do + mod="$(printf '%s\n' "${line}" | sed -E 's/[,/:].*//' | tr -d ' ')" + ver="$(printf '%s\n' "${line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" + case "${mod}" in nvidia*) ;; *) continue ;; esac + [ -n "${mod}" ] && [ -n "${ver}" ] || continue + log " dkms install ${mod}/${ver} -k ${KVER}" + dkms build -m "${mod}" -v "${ver}" -k "${KVER}" 2>/dev/null || true + dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force 2>/dev/null || true + done + # Belt-and-suspenders: try the autoinstaller pinned to the target kernel + # (ignored gracefully by older dkms that lack the -k flag). + dkms autoinstall -k "${KVER}" 2>/dev/null || true + log "DKMS status:"; dkms status 2>/dev/null || true +else + warn "dkms not found; relying on apt postinst build." +fi + +# --------------------------------------------------------------------------- +# 7. Verify the modules actually landed in the image kernel's module tree +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +if ls "${MODDIR}"/updates/dkms/nvidia*.ko* >/dev/null 2>&1 || \ + ls "${MODDIR}"/kernel/drivers/video/nvidia*.ko* >/dev/null 2>&1 || \ + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | grep -q .; then + log "Verified: nvidia kernel modules present under ${MODDIR}." + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | sed 's/^/ /' +else + die "no nvidia*.ko modules found under ${MODDIR} -- DKMS build did not \ +produce modules for the image kernel. Check that linux-headers-${KVER} and a \ +matching gcc are installed." +fi + +# --------------------------------------------------------------------------- +# 8. Optional: NVIDIA Fabric Manager (NVSwitch / HGX systems only) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then + FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${FM_PKG} ..." + if apt-get install -y --no-install-recommends "${FM_PKG}"; then + systemctl enable nvidia-fabricmanager.service 2>/dev/null || true + else + warn "could not install ${FM_PKG}; skipping fabric manager." + fi +fi + +# --------------------------------------------------------------------------- +# 9. Optional: nvidia-container-toolkit on the host +# (default OFF -- the GPU Operator ships and configures the toolkit) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Installing nvidia-container-toolkit on host ..." + install -d -m 0755 /usr/share/keyrings + if curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg; then + curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list + apt-get update && apt-get install -y --no-install-recommends nvidia-container-toolkit \ + || warn "nvidia-container-toolkit install failed." + else + warn "could not fetch nvidia-container-toolkit gpg key; skipping." + fi +fi + +# --------------------------------------------------------------------------- +# 10. Host module configuration: blacklist nouveau + autoload nvidia +# --------------------------------------------------------------------------- +log "Configuring nouveau blacklist and nvidia module autoload ..." +cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +blacklist nouveau +blacklist lbm-nouveau +options nouveau modeset=0 +alias nouveau off +alias lbm-nouveau off +EOF + +cat > /etc/modules-load.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Load the NVIDIA stack at boot so the GPU Operator sees a ready driver. +nvidia +nvidia_uvm +nvidia_modeset +nvidia_drm +EOF + +# NVIDIA driver run-time module options recommended for datacenter use: +# NVreg_OpenRmEnableUnsupportedGpus is only relevant for the open modules. +cat > /etc/modprobe.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +options nvidia NVreg_PreserveVideoMemoryAllocations=1 +EOF + +# Enable the persistence daemon (recommended for datacenter GPUs). +systemctl enable nvidia-persistenced.service 2>/dev/null || true + +# --------------------------------------------------------------------------- +# 11. depmod for the target kernel so modprobe can resolve nvidia at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 12. Rebuild the initrd so the nouveau blacklist applies in early boot +# --------------------------------------------------------------------------- +if [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed; nouveau blacklist still applies post-switchroot." + fi +elif [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 13. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. NVIDIA driver ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE}) baked in for kernel ${KVER}." +log "Reminder: install the GPU Operator with 'driver.enabled=false'." +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Reminder: you pre-installed the container toolkit -> also set 'toolkit.enabled=false'." +fi From aecf843b1d1fe60ec6553ce1409f6f46009bfc1b Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 17:18:24 -0700 Subject: [PATCH 02/34] docs(gpu): clarify relationship to NVIDIA's Local Package Repository section Explain that linux-headers/linux-image/linux-modules listed by NVIDIA belong to the driver-container + local-apt-mirror strategy (driver.enabled=true), whereas this integration uses the pre-installed-driver model (driver.enabled=false, no mirror) and satisfies those three requirements at build time in the image. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/nvidia-gpu-airgapped.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 1ba2956a..9ce4d2c3 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -48,6 +48,38 @@ If you also opt in to pre-installing the container toolkit on the host --- +## Relationship to NVIDIA's "Local Package Repository" section + +The NVIDIA air-gapped guide lists these Ubuntu packages under +**Local Package Repository → Required Packages**: + +``` +ubuntu: + linux-headers-${KERNEL_VERSION} + linux-image-${KERNEL_VERSION} + linux-modules-${KERNEL_VERSION} +``` + +That list belongs to the **driver-container** strategy: the node runs the GPU +Operator's *driver container*, which compiles the driver **at runtime** and pulls +those OS packages from **a local Ubuntu apt mirror you host**. It requires +`driver.enabled=true` plus a maintained mirror. + +This CanvOS integration deliberately uses the **other** supported strategy — +**pre-installed driver in the OS image** (`driver.enabled=false`) — so **no local +apt mirror is needed**. The substance of those three packages is still satisfied, +just at build time inside the image rather than from a runtime mirror: + +| NVIDIA-required package | How this integration satisfies it | +| --- | --- | +| `linux-headers-${KERNEL_VERSION}` | Installed at build time by `install-kernel-headers.sh` (ABI-exact; DKMS builds against these). | +| `linux-image-${KERNEL_VERSION}` | Already shipped in the Kairos base image (the bootable kernel). | +| `linux-modules-${KERNEL_VERSION}` | Already shipped in the Kairos base image (`/lib/modules/${KERNEL_VERSION}/`). | + +If you specifically want the driver-container + local-mirror model instead, this +script is not the right tool — you would host an apt mirror serving the packages +above and leave `driver.enabled=true`. + ## The key build-time problem this solves Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, **not** From e7a660e03b9a53944d548073dee17cd6ec7769fd Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 17:21:17 -0700 Subject: [PATCH 03/34] docs(gpu): document Ubuntu 24.04 support and driver-branch versioning semantics Clarify that the installer is version-agnostic (derives CUDA repo tag and kernel codename from the image's os-release), so it works on both Ubuntu 22.04 and 24.04. Note that NVIDIA_DRIVER_BRANCH selects a driver branch (latest patch within it), not an exact point release. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/nvidia-gpu-airgapped.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 9ce4d2c3..3c85eccd 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -10,6 +10,11 @@ in a **fully air-gapped** environment — with **no host-side network access** a - Wired into the `base-image` target in the [`Earthfile`](../Earthfile), gated by `INSTALL_NVIDIA_GPU_DRIVERS=true`. +**Supported targets:** Ubuntu **22.04** and **24.04**, `amd64`. The script is +version-agnostic — it derives the CUDA repo tag (`ubuntu2204` / `ubuntu2404`) and +the kernel codename (`jammy` / `noble`) from the image's `/etc/os-release` at +build time, so the same script works for both without changes. + --- ## Why do this (the split of responsibilities) @@ -108,7 +113,7 @@ It runs in the `base-image` target **after** the kernel is finalized ```sh OS_DISTRIBUTION=ubuntu - OS_VERSION=22.04 + OS_VERSION=22 # or 24 for Ubuntu 24.04 ARCH=amd64 INSTALL_NVIDIA_GPU_DRIVERS=true @@ -145,7 +150,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | Variable | Default | Description | | --- | --- | --- | | `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | -| `NVIDIA_DRIVER_BRANCH` | `570` | Driver branch to install (e.g. `550`, `570`, `580`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | +| `NVIDIA_DRIVER_BRANCH` | `570` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `570.86.15`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `proprietary` | `proprietary` or `open`. `open` uses the NVIDIA open GPU kernel modules (Turing architecture and newer only). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | | `NVIDIA_INSTALL_FABRICMANAGER` | `false` | Set `true` for NVSwitch / HGX systems (installs and enables `nvidia-fabricmanager`). | From b5196b7bf2d47f38e6c866fe5a272160148bf10d Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 17:37:31 -0700 Subject: [PATCH 04/34] feat(gpu): add AMD Instinct driver pre-install for air-gapped AMD GPU Operator Add scripts/install-amdgpu-drivers.sh, the AMD counterpart to the NVIDIA installer. Bakes the amdgpu-dkms kernel driver into the Ubuntu base image so AMD Instinct nodes can run the AMD GPU Operator with driver.enable=false in fully air-gapped environments (no host-side network, no operator-built driver). Reuses the same build-time approach: derive the image kernel from /lib/modules (not the builder's uname -r), install ABI-exact headers via install-kernel-headers.sh plus linux-modules-extra, register the AMD driver repo via the version-matched amdgpu-install package (auto-discovered from repo.radeon.com/amdgpu-install//), install amdgpu-dkms, force DKMS build/install/depmod against the image kernel, and verify amdgpu.ko landed. No nouveau-style blacklist is needed since amdgpu-dkms replaces the in-tree amdgpu module. Wire it into the base-image target behind INSTALL_AMD_GPU_DRIVERS with an AMDGPU_ROCM_VERSION arg (default 7.2.4), alongside a build-time guard making the NVIDIA and AMD pre-install paths mutually exclusive. Document in .arg.template, docs/amd-gpu-airgapped.md, and cross-link from the NVIDIA doc. Co-Authored-By: Claude Opus 4.8 (1M context) --- .arg.template | 8 + Earthfile | 25 +++ docs/amd-gpu-airgapped.md | 147 ++++++++++++++++++ docs/nvidia-gpu-airgapped.md | 4 + scripts/install-amdgpu-drivers.sh | 249 ++++++++++++++++++++++++++++++ 5 files changed, 433 insertions(+) create mode 100644 docs/amd-gpu-airgapped.md create mode 100755 scripts/install-amdgpu-drivers.sh diff --git a/.arg.template b/.arg.template index 4460591d..fb10e030 100644 --- a/.arg.template +++ b/.arg.template @@ -43,3 +43,11 @@ FORCE_INTERACTIVE_INSTALL=false # NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems # NVIDIA_INSTALL_CONTAINER_TOOLKIT=false # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false) # NVIDIA_REBUILD_INITRD=true # Rebuild initrd so the nouveau blacklist applies at early boot + +# AMD Instinct GPU driver pre-install (for running the AMD GPU Operator in air-gapped +# environments with driver.enable=false). Bakes the amdgpu-dkms kernel driver into the +# Ubuntu base image so GPU nodes need no host-side network. Mutually exclusive with the +# NVIDIA options above. See scripts/install-amdgpu-drivers.sh for details. +# INSTALL_AMD_GPU_DRIVERS=true +# AMDGPU_ROCM_VERSION=7.2.4 # ROCm/driver release; must match your bundled operator images (see repo.radeon.com/amdgpu-install/) +# AMDGPU_REBUILD_INITRD=true # Rebuild initrd for the image kernel diff --git a/Earthfile b/Earthfile index ed34f655..2360c3ce 100644 --- a/Earthfile +++ b/Earthfile @@ -74,6 +74,19 @@ ARG NVIDIA_INSTALL_FABRICMANAGER=false ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false ARG NVIDIA_REBUILD_INITRD=true +# AMD Instinct GPU driver pre-install (for air-gapped AMD GPU Operator with +# driver.enable=false). When true, the amdgpu-dkms kernel driver is baked into +# the Ubuntu base image so GPU nodes need no host-side network at boot. +ARG INSTALL_AMD_GPU_DRIVERS=false +ARG AMDGPU_ROCM_VERSION=7.2.4 +ARG AMDGPU_REBUILD_INITRD=true + +# NVIDIA and AMD driver pre-install are mutually exclusive within a single image. +IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] && [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + RUN echo "ERROR: INSTALL_NVIDIA_GPU_DRIVERS and INSTALL_AMD_GPU_DRIVERS are mutually exclusive. Enable only one." >&2 && \ + exit 1 +END + IF [ "$FIPS_ENABLED" = "true" ] && [ "$UPDATE_KERNEL" = "true" ] RUN echo "ERROR: UPDATE_KERNEL and FIPS_ENABLED are mutually exclusive. Cannot set both to true." >&2 && \ exit 1 @@ -825,6 +838,18 @@ base-image: rm -f /tmp/install-nvidia-drivers.sh /tmp/install-kernel-headers.sh END + # AMD Instinct GPU driver (amdgpu-dkms) + kernel module, built against the + # now-finalized image kernel. Mutually exclusive with the NVIDIA block above. + IF [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_ROCM_VERSION="$AMDGPU_ROCM_VERSION" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + END + IF [ "$CIS_HARDENING" = "true" ] COPY cis-harden/harden.sh /tmp/harden.sh RUN /tmp/harden.sh && rm /tmp/harden.sh diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md new file mode 100644 index 00000000..79deb88f --- /dev/null +++ b/docs/amd-gpu-airgapped.md @@ -0,0 +1,147 @@ +# Pre-installing the AMD Instinct GPU driver for air-gapped GPU Operator + +This guide explains how to bake the AMD **amdgpu** kernel-mode driver **into a +CanvOS Ubuntu base image**, so that AMD Instinct GPU nodes can run the +[AMD GPU Operator](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/specialized_networks/airgapped-install.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator building/managing the driver**. + +It is the AMD counterpart of [`nvidia-gpu-airgapped.md`](./nvidia-gpu-airgapped.md) +and follows the same "pre-installed driver" model. + +- Script: [`scripts/install-amdgpu-drivers.sh`](../scripts/install-amdgpu-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_AMD_GPU_DRIVERS=true`. + +**Supported targets:** Ubuntu **22.04** (jammy) and **24.04** (noble), `amd64`. +The codename is derived from the image at build time. + +> **Mutually exclusive with NVIDIA.** A single image supports one GPU vendor. +> Enabling both `INSTALL_AMD_GPU_DRIVERS` and `INSTALL_NVIDIA_GPU_DRIVERS` fails +> the build. + +--- + +## Split of responsibilities + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| amdgpu kernel module (`amdgpu`) + firmware | **In the OS image** | **This script (build time)** | +| ROCm user-space, device-plugin, node-labeller, metrics-exporter | Container images | AMD GPU Operator (from your content bundle) | + +The OS carries only the kernel driver; everything else is a container image you +mirror into your Palette content bundle. With `driver.enable=false` the operator +"directly uses inbox or pre-installed AMD GPU drivers" and only deploys the +device-plugin / node-labeller / metrics-exporter. + +At Helm-install time you **must** set: + +``` +--set driver.enable=false # note: "enable", not "enabled" +``` + +--- + +## Relationship to the AMD air-gapped guide + +The AMD guide's `driver.enable=true` path has the operator build the out-of-tree +driver at runtime, which needs build packages and (in restricted networks) a +local package mirror. This integration uses the opposite path +(`driver.enable=false`): the `amdgpu-dkms` driver is compiled into the image at +build time, so **no host-side mirror or network is needed at boot**. AMD's host +package requirements (`linux-headers-$(uname -r)`, `linux-modules-extra-...`, +`amdgpu-dkms`) are all satisfied at build time inside the image. + +--- + +## The key build-time problem this solves + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, not +the kernel baked into the image. The script therefore: + +1. derives the **target kernel** from `/lib/modules/*`, +2. installs **ABI-exact kernel headers** for it (reusing + [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh)) plus + `linux-modules-extra-`, +3. registers the AMD driver apt repo via the version-matched `amdgpu-install` + package (auto-discovered from `repo.radeon.com/amdgpu-install//`), +4. installs `amdgpu-dkms` and forces **DKMS build + install + `depmod`** against + the target kernel, and +5. **verifies** `amdgpu.ko` landed under `/lib/modules//`, failing + the build loudly otherwise. + +It runs in the `base-image` target **after** the kernel is finalized. + +> **No blacklist needed.** Unlike NVIDIA (where `nouveau` must be blacklisted), +> `amdgpu-dkms` replaces the in-tree `amdgpu` module of the same name; `depmod` +> prefers the DKMS copy. The script just autoloads `amdgpu`. + +--- + +## Quick start + +1. Edit `.arg` and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22 # or 24 for Ubuntu 24.04 + ARCH=amd64 + + INSTALL_AMD_GPU_DRIVERS=true + AMDGPU_ROCM_VERSION=7.2.4 + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_AMD_GPU_DRIVERS=true \ + --AMDGPU_ROCM_VERSION=6.4.4 + ``` + +3. Mirror the AMD GPU Operator container images into your Palette content bundle + and install the operator with `driver.enable=false`. + +--- + +## Configuration reference + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Bakes `amdgpu-dkms` into the Ubuntu base image. | +| `AMDGPU_ROCM_VERSION` | `7.2.4` | ROCm/driver release to install. Selects the `amdgpu-install` package under `repo.radeon.com/amdgpu-install//`, which configures the matching driver repo. **Must match the ROCm version of the operator images you bundle.** | +| `AMDGPU_REBUILD_INITRD` | `true` | Rebuild the initrd for the image kernel. | + +### Choosing a version + +Browse available releases at + (e.g. `7.2.4`, `6.4.4`, `6.2.2`). Pick +the one matching your hardware and the operator images in your bundle. + +--- + +## Verify on a booted node + +```sh +lsmod | grep amdgpu +dmesg | grep -i amdgpu +ls /sys/class/kfd 2>/dev/null && echo "KFD present" +# If you also bundle ROCm user-space tooling: +# rocminfo ; amd-smi list +``` + +--- + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path** (unsigned DKMS modules + won't load). Use the standard (non-UKI) Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** +- **Version alignment is yours to own** — `AMDGPU_ROCM_VERSION` must line up with + the ROCm version of the operator images you bundle. diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 3c85eccd..c69dc84a 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -15,6 +15,10 @@ version-agnostic — it derives the CUDA repo tag (`ubuntu2204` / `ubuntu2404`) the kernel codename (`jammy` / `noble`) from the image's `/etc/os-release` at build time, so the same script works for both without changes. +> For AMD Instinct GPUs, see [`amd-gpu-airgapped.md`](./amd-gpu-airgapped.md). +> The two are **mutually exclusive** — enabling both `INSTALL_NVIDIA_GPU_DRIVERS` +> and `INSTALL_AMD_GPU_DRIVERS` fails the build. + --- ## Why do this (the split of responsibilities) diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh new file mode 100755 index 00000000..cb6b611c --- /dev/null +++ b/scripts/install-amdgpu-drivers.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# +# install-amdgpu-drivers.sh +# +# Pre-install the AMD Instinct GPU kernel-mode driver (amdgpu-dkms) INTO a +# CanvOS / Kairos Ubuntu base image, so a node booted from the image can run +# the AMD GPU Operator in a fully air-gapped environment WITHOUT any host-side +# network access and WITHOUT the operator building/managing the driver. +# +# WHAT THIS COVERS (OS side only) +# ------------------------------- +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers that match the kernel shipped in the image +# (delegated to scripts/install-kernel-headers.sh) +# * linux-modules-extra for the image kernel (amdgpu pulls modules from it) +# * the AMD amdgpu kernel module built with DKMS against the IMAGE kernel +# (amdgpu-dkms + amdgpu-dkms-firmware) +# * amdgpu module autoload + initrd refresh +# +# WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, +# deployed by the AMD GPU Operator itself): +# * ROCm user-space, device-plugin, node-labeller, metrics exporter, etc. +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enable=false # note: "enable", not "enabled" +# (the operator then "directly uses inbox or pre-installed AMD GPU drivers" +# and only deploys device-plugin / node-labeller / metrics-exporter) +# +# WHY THE DKMS DANCE (same rationale as install-nvidia-drivers.sh) +# --------------------------------------------------------------- +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. We derive the target kernel from /lib/modules (the +# kernel that will actually boot) and force the DKMS build + module install + +# depmod against THAT kernel. +# +# NOTE ON BLACKLISTING +# -------------------- +# Unlike NVIDIA (where the open-source `nouveau` driver must be blacklisted), +# amdgpu-dkms REPLACES the in-tree `amdgpu` module (same module name). depmod +# prefers the updates/dkms copy, so there is nothing to blacklist -- we simply +# ensure amdgpu autoloads. +# +# CONNECTIVITY +# ------------ +# This script runs at BUILD time, where the builder has internet. It bakes +# everything into the image. The resulting image needs no network at boot. +# +# TUNABLES (environment variables; all optional) +# AMDGPU_ROCM_VERSION ROCm/driver release to install (e.g. 7.2.4, 6.4.4, +# 6.2.2). Default: 7.2.4 +# Must match the ROCm version of the operator images +# you bundle. This selects the amdgpu-install package +# under repo.radeon.com/amdgpu-install//, +# which configures the matching driver apt repo. +# AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. +# Default: true +# +set -u + +log() { echo "[install-amdgpu-drivers] $*"; } +warn() { echo "[install-amdgpu-drivers] WARNING: $*" >&2; } +die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +AMDGPU_ROCM_VERSION="${AMDGPU_ROCM_VERSION:-7.2.4}" +AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-true}" + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "AMD ROCm/driver version: ${AMDGPU_ROCM_VERSION}" + +# Ubuntu release codename (jammy / noble) read from the image itself. +codename="" +osid="ubuntu" +if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + codename="${VERSION_CODENAME:-}" + osid="${ID:-ubuntu}" +fi +[ -n "${codename}" ] || die "could not determine Ubuntu codename from /etc/os-release." + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev initramfs-tools \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers + modules-extra matching the image kernel +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# amdgpu depends on modules that live in linux-modules-extra (e.g. for some +# PCIe / crypto / networking helpers). Best-effort -- may be absent if Ubuntu +# rotated the ABI out of the live mirror. +log "Installing linux-modules-extra-${KVER} (best-effort) ..." +apt-get install -y "linux-modules-extra-${KVER}" || \ + warn "linux-modules-extra-${KVER} not available; continuing." + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. Register the AMD driver repo via the amdgpu-install package +# The amdgpu-install .deb (AMD's blessed entry point) configures the correct +# versioned driver apt repo + GPG key for the requested ROCm release. Its +# filename carries a build number, so we auto-discover it from the directory +# listing rather than hardcoding it. +# --------------------------------------------------------------------------- +inst_dir="https://repo.radeon.com/amdgpu-install/${AMDGPU_ROCM_VERSION}/${osid}/${codename}" +log "Locating amdgpu-install package under ${inst_dir}/ ..." +deb_name="$(curl -fsSL "${inst_dir}/" 2>/dev/null \ + | grep -oE 'amdgpu-install_[0-9A-Za-z._-]+_all\.deb' | sort -u | tail -1)" +[ -n "${deb_name}" ] || die "could not find an amdgpu-install package for ROCm \ +${AMDGPU_ROCM_VERSION} / ${codename} at ${inst_dir}/. \ +Check available versions at https://repo.radeon.com/amdgpu-install/" + +log "Installing ${deb_name} (configures the AMD driver apt repo) ..." +wget -qO /tmp/amdgpu-install.deb "${inst_dir}/${deb_name}" \ + || die "failed to download ${deb_name}." +apt-get install -y /tmp/amdgpu-install.deb || die "failed to install amdgpu-install." +rm -f /tmp/amdgpu-install.deb +apt-get update || warn "apt-get update after adding the AMD repo failed." + +# --------------------------------------------------------------------------- +# 5. Install the kernel-mode driver only (amdgpu-dkms + firmware) +# --------------------------------------------------------------------------- +log "Installing amdgpu-dkms ..." +apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware \ + || apt-get install -y amdgpu-dkms \ + || die "failed to install amdgpu-dkms. \ +List available driver packages with: apt-cache search amdgpu-dkms" + +# --------------------------------------------------------------------------- +# 6. Build the DKMS module against the IMAGE kernel (not the build host) +# --------------------------------------------------------------------------- +if command -v dkms >/dev/null 2>&1; then + log "Building amdgpu DKMS module for kernel ${KVER} ..." + # `dkms status` differs across versions: + # dkms 2.x: "amdgpu, 6.16.13, 6.14.0-36-generic, x86_64: installed" + # dkms 3.x: "amdgpu/6.16.13, 6.14.0-36-generic, x86_64: installed" + # Extract module name (up to first , / or :) + first version-looking token. + dkms status 2>/dev/null | grep -i amdgpu | while read -r line; do + mod="$(printf '%s\n' "${line}" | sed -E 's/[,/:].*//' | tr -d ' ')" + ver="$(printf '%s\n' "${line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" + case "${mod}" in amdgpu*) ;; *) continue ;; esac + [ -n "${mod}" ] && [ -n "${ver}" ] || continue + log " dkms install ${mod}/${ver} -k ${KVER}" + dkms build -m "${mod}" -v "${ver}" -k "${KVER}" 2>/dev/null || true + dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force 2>/dev/null || true + done + # Belt-and-suspenders: autoinstaller pinned to the target kernel. + dkms autoinstall -k "${KVER}" 2>/dev/null || true + log "DKMS status:"; dkms status 2>/dev/null || true +else + warn "dkms not found; relying on apt postinst build." +fi + +# --------------------------------------------------------------------------- +# 7. Verify the module landed in the image kernel's module tree +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +if find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + log "Verified: amdgpu kernel module present under ${MODDIR}." + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' +else + die "no amdgpu.ko module found under ${MODDIR} -- DKMS build did not produce \ +a module for the image kernel. Check that linux-headers-${KVER} and a matching \ +gcc are installed." +fi + +# --------------------------------------------------------------------------- +# 8. Autoload amdgpu at boot (no blacklist needed -- dkms replaces the in-tree +# module of the same name) +# --------------------------------------------------------------------------- +log "Configuring amdgpu module autoload ..." +cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Load the AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + +# --------------------------------------------------------------------------- +# 9. depmod for the target kernel so modprobe resolves amdgpu at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 10. Rebuild the initrd for the target kernel +# --------------------------------------------------------------------------- +if [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed." + fi +elif [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 11. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. AMD amdgpu driver (ROCm ${AMDGPU_ROCM_VERSION}) baked in for kernel ${KVER}." +log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." From 8df2f02c043a6aa3558041cb819fc27f426ef829 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 18:07:09 -0700 Subject: [PATCH 05/34] docs(gpu): document how to build air-gapped content when driver is pre-installed Add a "Building the air-gapped content" section to both the NVIDIA and AMD guides. Clarify that pre-installing the driver only lets you drop the driver image (NVIDIA) or the KMM/kaniko/ubuntu build images (AMD) -- the rest of the operator still ships as container images that must be mirrored into the content bundle. Cover the easy-to-miss pieces: NVIDIA still needs container-toolkit (unless also pre-installed on host) and the CUDA validation image, plus the k3s/rke2 containerd config env overrides for Palette Edge; AMD needs cert-manager installed first, a DeviceConfig CR with driver.enable=false, and node labelling. Recommend enumerating exact image tags via 'helm template' and note Palette content-bundle additional-images handling. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/amd-gpu-airgapped.md | 53 ++++++++++++++++++++++++++++++++ docs/nvidia-gpu-airgapped.md | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md index 79deb88f..8461cc11 100644 --- a/docs/amd-gpu-airgapped.md +++ b/docs/amd-gpu-airgapped.md @@ -138,6 +138,59 @@ ls /sys/class/kfd 2>/dev/null && echo "KFD present" --- +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS removes the **driver-build** images (KMM & +friends). The operator still deploys the rest as containers, so those images — +plus cert-manager and a couple of non-image steps — must be handled. +**Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enable=false`? | +| --- | --- | +| `rocm/gpu-operator` (controller-manager) | Yes | +| `rocm/gpu-operator-utils` | Yes | +| `rocm/k8s-device-plugin` | Yes | +| `rocm/k8s-device-plugin:labeller-*` (node labeller) | Yes | +| `rocm/device-metrics-exporter` | Yes, if you want metrics | +| `rocm/device-config-manager` | Yes | +| `busybox:1.36` (init container) | Yes | +| `registry.k8s.io/nfd/node-feature-discovery` | Yes — unless the cluster already runs NFD | +| cert-manager (`controller`, `webhook`, `cainjector`, `acmesolver`) | Yes — hard dependency | +| KMM images (operator / webhook / worker / signimage) | **No — skip** | +| `gcr.io/kaniko-project/executor`, `ubuntu:` (driver build) | **No — skip** | +| `rocm/test-runner` | Optional (testing only) | + +Render the exact set from the chart rather than transcribing tags: + +```sh +helm template amd-gpu ./gpu-operator-.tgz -f operator-values.yaml \ + | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. **Install cert-manager first** (with its images pulled from your registry) — + the AMD operator will not start without it. +2. In the `DeviceConfig` CR, set `spec.driver.enable: false`. +3. Override every image (`controllerManager.manager.image`, + `commonConfig.initContainerImage`, `utilsContainer.image`, + `devicePlugin.devicePluginImage`, `devicePlugin.nodeLabellerImage`, + `metricsExporter.image`, `configManager.image`, and the NFD image) to your + bundle/registry; set `imagePullSecrets` as needed. +4. Ensure GPU nodes are labelled (via NFD or manually): + `feature.node.kubernetes.io/amd-gpu=true`. + +### Palette content bundle + +Add the AMD GPU Operator (and cert-manager) as Helm packs in the cluster profile, +then build the content bundle so it includes the rendered images above (minus the +KMM/kaniko/ubuntu build images). Images set only via `values.yaml` may need to be +added to the pack's additional-images list if the bundle builder doesn't +auto-detect them. Verify on a node with `lsmod | grep amdgpu` and by checking the +operator pods reach `Ready`. + ## Limitations / caveats - **Secure Boot / UKI is not supported by this path** (unsigned DKMS modules diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index c69dc84a..099ee78e 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -193,6 +193,65 @@ lsmod | grep nvidia --- +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS only removes the **driver image**. The +operator still deploys everything else as containers, so those images (and a few +non-image steps) must be handled. **Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enabled=false`? | +| --- | --- | +| `gpu-operator` | Yes | +| `gpu-operator-validator` | Yes | +| `container-toolkit` | Yes — **unless** you also pre-installed it on the host (`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true` → then set `toolkit.enabled=false` and skip this image) | +| `k8s-device-plugin` | Yes | +| `gpu-feature-discovery` | Yes | +| `dcgm` + `dcgm-exporter` | Yes, if you want GPU metrics | +| `node-feature-discovery` | Yes — unless the cluster already runs NFD (`nfd.enabled=false`) | +| CUDA validation image (`nvcr.io/nvidia/cuda:…`) | Yes — used by the validator init container (easy to miss) | +| `k8s-mig-manager` | Only if using MIG | +| **`driver`** | **No — skip it (that's the point of pre-installing)** | + +Don't transcribe tags by hand — they change per operator version. Render the +exact set from the chart and mirror precisely that: + +```sh +helm template gpu-operator nvidia/gpu-operator --version \ + --set driver.enabled=false | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. `--set driver.enabled=false`. +2. Override **every** image `repository` to your bundle/registry and set + `imagePullSecrets`. +3. **Palette Edge (k3s / rke2) gotcha:** the container-toolkit defaults assume + stock containerd. On k3s/rke2 you must point it at the right socket and + config, e.g.: + + ``` + --set toolkit.env[0].name=CONTAINERD_CONFIG \ + --set toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml \ + --set toolkit.env[1].name=CONTAINERD_SOCKET \ + --set toolkit.env[1].value=/run/k3s/containerd/containerd.sock \ + --set toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS \ + --set toolkit.env[2].value=nvidia + ``` + + (rke2 paths: `/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl`, + `/run/k3s/containerd/containerd.sock`.) Miss this and workloads never get the + GPU runtime even though the driver is present. + +### Palette content bundle + +Add the GPU Operator as a Helm pack in the cluster profile, then build the +content bundle so it includes the rendered images above (minus `driver`). Images +set only via `values.yaml` may need to be added to the pack's additional-images +list if the bundle builder doesn't auto-detect them. Verify on a node with +`nvidia-smi` and by checking the operator's `*-validator` pods reach `Ready`. + ## Limitations / caveats - **Secure Boot / UKI is not supported by this path.** When `IS_UKI=true`, DKMS From 63a8596a5c7057ff16b20b07c7cd918e32909663 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 9 Jul 2026 23:03:03 -0700 Subject: [PATCH 06/34] chore(gpu): default NVIDIA_DRIVER_BRANCH to 580 Bump the default NVIDIA driver branch from 570 to 580 across the Earthfile ARG, install-nvidia-drivers.sh, .arg.template, and docs. Refresh point-release examples to 580.159.03. The 570 entries left in "e.g. 550, 570, 580" lists are valid-branch examples, not defaults. Co-Authored-By: Claude Opus 4.8 (1M context) --- .arg.template | 2 +- Earthfile | 2 +- docs/nvidia-gpu-airgapped.md | 6 +++--- scripts/install-nvidia-drivers.sh | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.arg.template b/.arg.template index fb10e030..b77c5ef6 100644 --- a/.arg.template +++ b/.arg.template @@ -37,7 +37,7 @@ FORCE_INTERACTIVE_INSTALL=false # kernel modules into the Ubuntu base image so GPU nodes need no host-side network. # See scripts/install-nvidia-drivers.sh for details. # INSTALL_NVIDIA_GPU_DRIVERS=true -# NVIDIA_DRIVER_BRANCH=570 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') +# NVIDIA_DRIVER_BRANCH=580 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') # NVIDIA_DRIVER_TYPE=proprietary # proprietary | open (open GPU kernel modules; Turing+ only) # NVIDIA_USE_CUDA_REPO=true # Add NVIDIA CUDA network repo at build time (has every -server branch) # NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems diff --git a/Earthfile b/Earthfile index 2360c3ce..c76d9e3f 100644 --- a/Earthfile +++ b/Earthfile @@ -67,7 +67,7 @@ ARG UPDATE_KERNEL=false # When true, the NVIDIA data-center driver + DKMS kernel modules are baked into the # Ubuntu base image so GPU nodes need no host-side network at boot. ARG INSTALL_NVIDIA_GPU_DRIVERS=false -ARG NVIDIA_DRIVER_BRANCH=570 +ARG NVIDIA_DRIVER_BRANCH=580 ARG NVIDIA_DRIVER_TYPE=proprietary ARG NVIDIA_USE_CUDA_REPO=true ARG NVIDIA_INSTALL_FABRICMANAGER=false diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 099ee78e..2dd60274 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -121,7 +121,7 @@ It runs in the `base-image` target **after** the kernel is finalized ARCH=amd64 INSTALL_NVIDIA_GPU_DRIVERS=true - NVIDIA_DRIVER_BRANCH=570 # verify the branch exists (see below) + NVIDIA_DRIVER_BRANCH=580 # verify the branch exists (see below) NVIDIA_DRIVER_TYPE=proprietary # or: open (Turing+ only) ``` @@ -136,7 +136,7 @@ It runs in the `base-image` target **after** the kernel is finalized ```sh ./earthly.sh +base-image --ARCH=amd64 \ --INSTALL_NVIDIA_GPU_DRIVERS=true \ - --NVIDIA_DRIVER_BRANCH=570 \ + --NVIDIA_DRIVER_BRANCH=580 \ --NVIDIA_DRIVER_TYPE=proprietary ``` @@ -154,7 +154,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | Variable | Default | Description | | --- | --- | --- | | `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | -| `NVIDIA_DRIVER_BRANCH` | `570` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `570.86.15`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | +| `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `proprietary` | `proprietary` or `open`. `open` uses the NVIDIA open GPU kernel modules (Turing architecture and newer only). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | | `NVIDIA_INSTALL_FABRICMANAGER` | `false` | Set `true` for NVSwitch / HGX systems (installs and enables `nvidia-fabricmanager`). | diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 28e1844b..498e7a91 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -43,7 +43,7 @@ # # TUNABLES (environment variables; all optional) # NVIDIA_DRIVER_BRANCH Driver branch to install (e.g. 550, 570, 580). -# Default: 570 (a data-center production branch) +# Default: 580 (a data-center production branch) # NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: proprietary # (open = open GPU kernel modules; Turing+ only) # NVIDIA_USE_CUDA_REPO "true" to add developer.download.nvidia.com CUDA @@ -68,7 +68,7 @@ die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- -NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-570}" +NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-580}" NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-proprietary}" NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-false}" @@ -194,8 +194,8 @@ if command -v dkms >/dev/null 2>&1; then log "Building NVIDIA DKMS modules for kernel ${KVER} ..." # Explicitly (re)build every registered nvidia dkms module for the target. # `dkms status` output differs across versions: - # dkms 2.x: "nvidia, 570.86.15, 6.14.0-36-generic, x86_64: installed" - # dkms 3.x: "nvidia/570.86.15, 6.14.0-36-generic, x86_64: installed" + # dkms 2.x: "nvidia, 580.159.03, 6.14.0-36-generic, x86_64: installed" + # dkms 3.x: "nvidia/580.159.03, 6.14.0-36-generic, x86_64: installed" # Extract the module name (up to the first , / or :) and the first # version-looking token, which works for both formats. dkms status 2>/dev/null | grep -i nvidia | while read -r line; do From 98edc21029b6969690c59bbb110fc88d7d1db13a Mon Sep 17 00:00:00 2001 From: abhijithspectro Date: Fri, 10 Jul 2026 17:05:08 +0000 Subject: [PATCH 07/34] Modified install-amdgpu-drivers.sh for fixing build errors --- scripts/install-amdgpu-drivers.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index cb6b611c..9b6bca10 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -164,10 +164,23 @@ apt-get update || warn "apt-get update after adding the AMD repo failed." # 5. Install the kernel-mode driver only (amdgpu-dkms + firmware) # --------------------------------------------------------------------------- log "Installing amdgpu-dkms ..." -apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware \ - || apt-get install -y amdgpu-dkms \ - || die "failed to install amdgpu-dkms. \ +# Note: amdgpu-dkms post-install script may fail in container environments +# due to missing EFI support. We attempt the install and continue even if dpkg +# post-install fails, then manually fix the configuration. +apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware 2>&1 | grep -v "dpkg: error" || true +apt-get install -y amdgpu-dkms 2>&1 | grep -v "dpkg: error" || true + +# Force-configure any packages with broken post-install scripts +log "Force-configuring packages with broken installations..." +dpkg --configure -a --force-all 2>&1 || true + +# Verify amdgpu-dkms was at least partially installed +if ! dpkg -l | grep -q "amdgpu-dkms"; then + die "failed to install amdgpu-dkms. \ List available driver packages with: apt-cache search amdgpu-dkms" +fi + +log "amdgpu-dkms package installation completed (post-install script errors suppressed)." # --------------------------------------------------------------------------- # 6. Build the DKMS module against the IMAGE kernel (not the build host) From 1c3a0897c2e0d96103ae91b5e2cee90b044abb0e Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Fri, 10 Jul 2026 23:35:13 +0530 Subject: [PATCH 08/34] fix(gpu): block nouveau at kernel cmdline + in initrd to prevent boot hang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On modern NVIDIA data-center GPUs (Ada/Hopper/Blackwell — reproduced on HPE ProLiant Compute DL380a Gen12 with a GH100-class GPU), a CanvOS image built with INSTALL_NVIDIA_GPU_DRIVERS=true would hang forever at first boot inside systemd-udev-settle. Console showed a kernel stack trace ending at gh100_gsp_oneinit+0x2cf/0x440 [nouveau]: initramfs udev auto-loaded nouveau before the rootfs modprobe.d blacklist could apply, and nouveau's experimental GSP-RM path hung on the GPU. udev-settle never returned; the node never came up; nvidia.ko never got to load. Three changes, together sufficient to unbrick boot: * Earthfile: when INSTALL_NVIDIA_GPU_DRIVERS is true, append rd.driver.blacklist=nouveau modprobe.blacklist=nouveau nouveau.modeset=0 to /etc/cos/bootargs.cfg. This is the actual load-bearing fix: rd.driver.blacklist= is honored by dracut/initramfs *before* any udev event fires, so nouveau never binds. Mirrors the existing cgroup-v2 append pattern immediately above. * install-nvidia-drivers.sh: write /etc/dracut.conf.d/95-blacklist-nouveau.conf so BOTH the script's own dracut rebuild AND the later Earthfile-driven dracut rebuild (a) omit nouveau.ko from the initrd entirely and (b) carry the modprobe blacklist file into initramfs. Belt-and-braces in case the cmdline args are ever stripped downstream. * install-nvidia-drivers.sh: drop nvidia_drm from /etc/modules-load.d/nvidia.conf. On headless GPU nodes it serves no purpose and loading it at early boot has a class of KMS/DRM-conflict failure modes. Loads on demand if anything actually needs it. Follow-up: the UKI build path (--extend-cmdline in the build-uki targets) also needs the same nouveau cmdline args when INSTALL_NVIDIA_GPU_DRIVERS is true. Not addressed here since field failures were non-UKI grub boot; worth adding in a subsequent commit. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 11 +++++++++++ scripts/install-nvidia-drivers.sh | 25 +++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Earthfile b/Earthfile index c76d9e3f..9f0f027f 100644 --- a/Earthfile +++ b/Earthfile @@ -912,6 +912,17 @@ base-image: RUN if ! grep -Fq "systemd.unified_cgroup_hierarchy=1" /etc/cos/bootargs.cfg; then \ sed -i 's|\(set baseCmd="[^"]*\)"|\1 systemd.unified_cgroup_hierarchy=1"|' /etc/cos/bootargs.cfg; \ fi + + # When the NVIDIA driver is pre-installed, block nouveau at the kernel + # command line. modprobe.d blacklists don't apply until AFTER switchroot, + # by which point initramfs udev has already auto-loaded nouveau on + # modern data-center GPUs (Ada/Hopper/Blackwell) and hung in GSP init, + # stalling systemd-udev-settle indefinitely. + IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] + RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau modprobe.blacklist=nouveau nouveau.modeset=0"|' /etc/cos/bootargs.cfg; \ + fi + END END KAIROS_RELEASE: diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 498e7a91..987ff4e3 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -278,10 +278,11 @@ EOF cat > /etc/modules-load.d/nvidia.conf <<'EOF' # Managed by CanvOS install-nvidia-drivers.sh # Load the NVIDIA stack at boot so the GPU Operator sees a ready driver. +# nvidia_drm is intentionally omitted: it grabs KMS/DRM and on headless GPU +# nodes can wedge early boot. It loads on demand if anything wants it. nvidia nvidia_uvm nvidia_modeset -nvidia_drm EOF # NVIDIA driver run-time module options recommended for datacenter use: @@ -301,7 +302,27 @@ log "Running depmod -a ${KVER} ..." depmod -a "${KVER}" || warn "depmod reported an error." # --------------------------------------------------------------------------- -# 12. Rebuild the initrd so the nouveau blacklist applies in early boot +# 12. Make sure the initrd honors the nouveau blacklist and does NOT ship +# nouveau.ko. The /etc/modprobe.d/blacklist-nouveau.conf we wrote above +# lives on the rootfs and is only consulted AFTER switchroot; by then +# nouveau has already been auto-loaded by initramfs udev on modern +# NVIDIA data-center GPUs (Ada/Hopper/Blackwell), where nouveau's GSP-RM +# support hangs on device init and stalls udev-settle forever. +# +# Fix: write a dracut.conf.d snippet so BOTH this script's dracut +# rebuild AND the later Earthfile-driven dracut rebuild produce an +# initrd that (a) omits nouveau entirely and (b) carries the modprobe +# blacklist file, so initramfs modprobe honors it too. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/95-blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +omit_drivers+=" nouveau lbm-nouveau " +install_items+=" /etc/modprobe.d/blacklist-nouveau.conf " +EOF + +# --------------------------------------------------------------------------- +# 12b. Rebuild the initrd so the nouveau blacklist applies in early boot # --------------------------------------------------------------------------- if [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then log "Rebuilding initrd for ${KVER} (dracut) ..." From 0a75e9119c09c22ced06e2c429f5f290050e24f0 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Sat, 11 Jul 2026 14:45:37 +0530 Subject: [PATCH 09/34] chore(gpu): default NVIDIA_DRIVER_TYPE to open for Hopper/Blackwell support Flip the default flavor of the pre-installed NVIDIA kernel modules from "proprietary" to "open". The closed modules fail RmInitAdapter (0x22:0x56:897) on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); `nvidia-smi` on such a host reports "No devices were found" and the entire GPU Operator stack downstream stays wedged (toolkit init loops, containerd never registers the nvidia runtime, device plugin never advertises nvidia.com/gpu, workload pods stay Pending on Insufficient nvidia.com/gpu). The open modules also work fine on Turing, Ampere, and Ada, so the new default is safe for every server-class NVIDIA GPU generation Turing and newer -- essentially every card CanvOS is likely to target. Only Pascal / Volta (V100/P100/P40) require the closed modules and now need an explicit NVIDIA_DRIVER_TYPE=proprietary override. Applied consistently across: * scripts/install-nvidia-drivers.sh -- runtime default + header comment * Earthfile -- ARG default * .arg.template -- user-facing hint * docs/nvidia-gpu-airgapped.md -- config table plus new "Choosing the module flavor" section with a hardware/flavor matrix and the RmInitAdapter symptom description for people googling the error. Co-Authored-By: Claude Opus 4.7 --- .arg.template | 3 ++- Earthfile | 2 +- docs/nvidia-gpu-airgapped.md | 26 +++++++++++++++++++++++++- scripts/install-nvidia-drivers.sh | 13 ++++++++++--- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.arg.template b/.arg.template index b77c5ef6..6193425c 100644 --- a/.arg.template +++ b/.arg.template @@ -38,7 +38,8 @@ FORCE_INTERACTIVE_INSTALL=false # See scripts/install-nvidia-drivers.sh for details. # INSTALL_NVIDIA_GPU_DRIVERS=true # NVIDIA_DRIVER_BRANCH=580 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') -# NVIDIA_DRIVER_TYPE=proprietary # proprietary | open (open GPU kernel modules; Turing+ only) +# NVIDIA_DRIVER_TYPE=open # open | proprietary. Default "open" (REQUIRED on Hopper/Blackwell, +# safe on Turing/Ampere/Ada). Set "proprietary" only for pre-Turing GPUs. # NVIDIA_USE_CUDA_REPO=true # Add NVIDIA CUDA network repo at build time (has every -server branch) # NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems # NVIDIA_INSTALL_CONTAINER_TOOLKIT=false # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false) diff --git a/Earthfile b/Earthfile index 9f0f027f..2db37a46 100644 --- a/Earthfile +++ b/Earthfile @@ -68,7 +68,7 @@ ARG UPDATE_KERNEL=false # Ubuntu base image so GPU nodes need no host-side network at boot. ARG INSTALL_NVIDIA_GPU_DRIVERS=false ARG NVIDIA_DRIVER_BRANCH=580 -ARG NVIDIA_DRIVER_TYPE=proprietary +ARG NVIDIA_DRIVER_TYPE=open ARG NVIDIA_USE_CUDA_REPO=true ARG NVIDIA_INSTALL_FABRICMANAGER=false ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 2dd60274..dd462068 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -155,7 +155,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | --- | --- | --- | | `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | | `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | -| `NVIDIA_DRIVER_TYPE` | `proprietary` | `proprietary` or `open`. `open` uses the NVIDIA open GPU kernel modules (Turing architecture and newer only). | +| `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | | `NVIDIA_INSTALL_FABRICMANAGER` | `false` | Set `true` for NVSwitch / HGX systems (installs and enables `nvidia-fabricmanager`). | | `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | @@ -173,6 +173,30 @@ apt-cache search 'nvidia-headless-.*-server' Pick a branch supported by both your GPU generation and the CUDA/toolkit versions of the operator images you're bundling. +### Choosing the module flavor (`NVIDIA_DRIVER_TYPE`) + +The default is `open`. It works on every server GPU generation Turing and newer, +and is **required** for Hopper and Blackwell. Override to `proprietary` only for +pre-Turing hardware. + +| GPU generation | Example cards | Required `NVIDIA_DRIVER_TYPE` | +| -------------- | ------------------------------------------------------ | ----------------------------- | +| Blackwell | RTX PRO 6000 Blackwell, B100, B200, GB200 | `open` (only) | +| Hopper | H100, H200 | `open` (only) | +| Ada Lovelace | L4, L40, L40S, RTX 6000 Ada | either (`open` recommended) | +| Ampere | A100, A10, A30, A40 | either | +| Turing | T4, RTX 20xx | either | +| Pre-Turing | V100, P100, P40 | `proprietary` (only) | + +Symptom of the wrong choice on Hopper/Blackwell: `nvidia-smi` reports +`No devices were found`, and `dmesg` shows one line per GPU of the form +`NVRM: GPU : RmInitAdapter failed! (0x22:0x56:897)`. In that state the +GPU Operator's toolkit init container loops on +`Attempting to validate a driver container installation`, containerd never +registers the `nvidia` runtime handler, the device plugin never advertises +`nvidia.com/gpu`, and workload pods stay `Pending` on +`Insufficient nvidia.com/gpu`. + --- ## What the script configures on the host diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 987ff4e3..70e4d596 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -44,8 +44,15 @@ # TUNABLES (environment variables; all optional) # NVIDIA_DRIVER_BRANCH Driver branch to install (e.g. 550, 570, 580). # Default: 580 (a data-center production branch) -# NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: proprietary -# (open = open GPU kernel modules; Turing+ only) +# NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: open +# "open" is REQUIRED on Hopper (H100/H200) and +# Blackwell (RTX PRO 6000 Blackwell, B100, B200, +# GB200) — the closed modules fail with +# "RmInitAdapter (0x22:0x56:897)" on those GPUs +# and `nvidia-smi` reports "No devices were found". +# Also safe on Turing/Ampere/Ada. Override to +# "proprietary" only for pre-Turing hardware +# (Pascal/Volta). # NVIDIA_USE_CUDA_REPO "true" to add developer.download.nvidia.com CUDA # repo (recommended, has every -server branch). # "false" to use only Ubuntu's own repos. @@ -69,7 +76,7 @@ die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } # Config # --------------------------------------------------------------------------- NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-580}" -NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-proprietary}" +NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-open}" NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-false}" NVIDIA_INSTALL_CONTAINER_TOOLKIT="${NVIDIA_INSTALL_CONTAINER_TOOLKIT:-false}" From eb339f0d8a20b1c07d2fdce21ebcae1c956c33ec Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 11:27:55 -0700 Subject: [PATCH 10/34] feat(gpu): add inbox driver-source mode + fail-hard AMD DKMS install The pre-install script silently shipped broken images when AMD's amdgpu-dkms source did not build against the image kernel (observed on Ubuntu 24.04 HWE 6.17 with the previous 7.2.4 / 30.30.4 default, whose 6.16.13 source fails at DKMS configure). The verify step accepted the in-tree amdgpu.ko.zst as sufficient because it only searched anywhere under /lib/modules/$KVER/, and apt-get failures were papered over. Nodes booted using the in-tree driver instead of the DKMS one without any signal that this had happened. Changes: - Introduce AMDGPU_DRIVER_SOURCE={dkms,inbox}. dkms (default) DKMS-builds AMD's out-of-tree amdgpu; inbox skips the AMD apt repo entirely and only ensures the kernel's in-tree amdgpu autoloads. Any other value is a hard error. - Rename AMDGPU_ROCM_VERSION -> AMDGPU_DRIVER_RELEASE (the URL segment is a driver-release marker, not a ROCm x.y.z). Bump default 7.2.4 -> 31.30 (amdgpu-dkms 6.19.4, covers 22.04/24.04 kernels through 6.17). - Fail hard in dkms mode: set -eo pipefail; remove '|| true' shims around dkms build/install; drop 'dpkg --configure -a --force-all' silent-fallback; verify amdgpu landed under $MODDIR/updates/dkms AND that dkms status shows ': installed' for the target kernel. Failure prints make.log tail and points at bumping AMDGPU_DRIVER_RELEASE or switching to inbox. - Write /etc/canvos/amdgpu-driver-source on-node marker so ops can query which mode ran and, in dkms mode, the release + module/version + kernel. - Docs: two-modes explainer up front, snapshot mapping table (2026-07-11) linking to AMD's canonical system-requirements page, "when the DKMS build fails" section documenting the bump-or-fallback flow. Co-Authored-By: Claude Opus 4.7 --- .arg.template | 18 +- Earthfile | 16 +- docs/amd-gpu-airgapped.md | 121 ++++++++++--- scripts/install-amdgpu-drivers.sh | 276 ++++++++++++++++++++---------- 4 files changed, 307 insertions(+), 124 deletions(-) diff --git a/.arg.template b/.arg.template index 6193425c..a89a2e9b 100644 --- a/.arg.template +++ b/.arg.template @@ -46,9 +46,19 @@ FORCE_INTERACTIVE_INSTALL=false # NVIDIA_REBUILD_INITRD=true # Rebuild initrd so the nouveau blacklist applies at early boot # AMD Instinct GPU driver pre-install (for running the AMD GPU Operator in air-gapped -# environments with driver.enable=false). Bakes the amdgpu-dkms kernel driver into the -# Ubuntu base image so GPU nodes need no host-side network. Mutually exclusive with the -# NVIDIA options above. See scripts/install-amdgpu-drivers.sh for details. +# environments with driver.enable=false). Mutually exclusive with the NVIDIA options +# above. See scripts/install-amdgpu-drivers.sh and docs/amd-gpu-airgapped.md for details. # INSTALL_AMD_GPU_DRIVERS=true -# AMDGPU_ROCM_VERSION=7.2.4 # ROCm/driver release; must match your bundled operator images (see repo.radeon.com/amdgpu-install/) +# AMDGPU_DRIVER_SOURCE=dkms # dkms | inbox. "dkms" bakes AMD's amdgpu-dkms +# # built against the image kernel (recommended +# # for Instinct/MI silicon). "inbox" uses the +# # in-tree amdgpu module from linux-modules-* +# # and skips the AMD apt repo entirely — use +# # when the DKMS build fails against your image +# # kernel (see docs). +# AMDGPU_DRIVER_RELEASE=31.30 # Only used with dkms mode. amdgpu-install +# # release marker (URL segment under +# # repo.radeon.com/amdgpu-install//, e.g. +# # 31.30, 31.10, 30.30.4). See the version- +# # pick table in docs/amd-gpu-airgapped.md. # AMDGPU_REBUILD_INITRD=true # Rebuild initrd for the image kernel diff --git a/Earthfile b/Earthfile index 2db37a46..5197bb02 100644 --- a/Earthfile +++ b/Earthfile @@ -75,10 +75,17 @@ ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false ARG NVIDIA_REBUILD_INITRD=true # AMD Instinct GPU driver pre-install (for air-gapped AMD GPU Operator with -# driver.enable=false). When true, the amdgpu-dkms kernel driver is baked into -# the Ubuntu base image so GPU nodes need no host-side network at boot. +# driver.enable=false). See scripts/install-amdgpu-drivers.sh + docs/amd-gpu-airgapped.md. ARG INSTALL_AMD_GPU_DRIVERS=false -ARG AMDGPU_ROCM_VERSION=7.2.4 +# dkms | inbox. "dkms" builds AMD's amdgpu-dkms against the image kernel (default, +# recommended for Instinct silicon). "inbox" uses the in-tree amdgpu module shipped +# with linux-modules-* and skips the AMD apt repo — use only when the DKMS build +# fails against your image kernel and you accept the in-tree driver's feature set. +ARG AMDGPU_DRIVER_SOURCE=dkms +# amdgpu-install release marker (URL segment under repo.radeon.com/amdgpu-install//). +# Default 31.30 ships amdgpu-dkms 6.19.4, which builds against Linux kernels through +# 6.17 (Ubuntu 22.04 & 24.04 HWE range). See docs/amd-gpu-airgapped.md for the mapping. +ARG AMDGPU_DRIVER_RELEASE=31.30 ARG AMDGPU_REBUILD_INITRD=true # NVIDIA and AMD driver pre-install are mutually exclusive within a single image. @@ -844,7 +851,8 @@ base-image: COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ - AMDGPU_ROCM_VERSION="$AMDGPU_ROCM_VERSION" \ + AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ /tmp/install-amdgpu-drivers.sh && \ rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md index 8461cc11..67c01383 100644 --- a/docs/amd-gpu-airgapped.md +++ b/docs/amd-gpu-airgapped.md @@ -1,7 +1,7 @@ # Pre-installing the AMD Instinct GPU driver for air-gapped GPU Operator -This guide explains how to bake the AMD **amdgpu** kernel-mode driver **into a -CanvOS Ubuntu base image**, so that AMD Instinct GPU nodes can run the +This guide explains how to pre-provision the AMD **amdgpu** kernel-mode driver +**in a CanvOS Ubuntu base image**, so that AMD Instinct GPU nodes can run the [AMD GPU Operator](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/specialized_networks/airgapped-install.html) in a **fully air-gapped** environment — with **no host-side network access** and **without the operator building/managing the driver**. @@ -16,6 +16,17 @@ and follows the same "pre-installed driver" model. **Supported targets:** Ubuntu **22.04** (jammy) and **24.04** (noble), `amd64`. The codename is derived from the image at build time. +## Two driver-source modes (`AMDGPU_DRIVER_SOURCE`) + +| Mode | What ships in the image | When to use | +| --- | --- | --- | +| `dkms` (default) | AMD's `amdgpu-dkms` source is DKMS-built against the image kernel and lands under `/lib/modules//updates/dkms/`. | Recommended for Instinct/MI silicon. The AMD out-of-tree driver typically carries newer SMU firmware interfaces and per-SKU support ahead of what the in-tree amdgpu has. | +| `inbox` | No AMD apt repo is added; the script only ensures the in-tree `amdgpu` module (shipped in `linux-modules-`) autoloads. | Fallback when the DKMS build fails against your image kernel — e.g. AMD hasn't yet published a driver release whose source builds against a very new kernel. Requires accepting the in-tree driver's feature set. | + +Both modes still require `driver.enable=false` at the Helm layer — the operator +does not build a driver either way. A marker at +`/etc/canvos/amdgpu-driver-source` on the booted node records which mode ran. + > **Mutually exclusive with NVIDIA.** A single image supports one GPU vendor. > Enabling both `INSTALL_AMD_GPU_DRIVERS` and `INSTALL_NVIDIA_GPU_DRIVERS` fails > the build. @@ -47,34 +58,41 @@ At Helm-install time you **must** set: The AMD guide's `driver.enable=true` path has the operator build the out-of-tree driver at runtime, which needs build packages and (in restricted networks) a local package mirror. This integration uses the opposite path -(`driver.enable=false`): the `amdgpu-dkms` driver is compiled into the image at -build time, so **no host-side mirror or network is needed at boot**. AMD's host -package requirements (`linux-headers-$(uname -r)`, `linux-modules-extra-...`, -`amdgpu-dkms`) are all satisfied at build time inside the image. +(`driver.enable=false`): the driver is either DKMS-built into the image at +build time (`AMDGPU_DRIVER_SOURCE=dkms`) or the in-tree amdgpu is used as-is +(`AMDGPU_DRIVER_SOURCE=inbox`). Either way, **no host-side mirror or network is +needed at boot**. --- -## The key build-time problem this solves +## The key build-time problem this solves (dkms mode) Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, not -the kernel baked into the image. The script therefore: +the kernel baked into the image. In `dkms` mode the script therefore: 1. derives the **target kernel** from `/lib/modules/*`, 2. installs **ABI-exact kernel headers** for it (reusing [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh)) plus `linux-modules-extra-`, -3. registers the AMD driver apt repo via the version-matched `amdgpu-install` - package (auto-discovered from `repo.radeon.com/amdgpu-install//`), +3. registers the AMD driver apt repo via the release-matched `amdgpu-install` + package (auto-discovered from `repo.radeon.com/amdgpu-install//`), 4. installs `amdgpu-dkms` and forces **DKMS build + install + `depmod`** against the target kernel, and -5. **verifies** `amdgpu.ko` landed under `/lib/modules//`, failing - the build loudly otherwise. +5. **verifies** an `amdgpu.ko` landed under + `/lib/modules//updates/dkms/` **and** that `dkms status` + reports the module as `installed` for that kernel — the in-tree module + shipped under `kernel/…` is *not* accepted. Failing either check aborts + the image build with a pointer at `AMDGPU_DRIVER_RELEASE` and the inbox + fallback. It runs in the `base-image` target **after** the kernel is finalized. > **No blacklist needed.** Unlike NVIDIA (where `nouveau` must be blacklisted), -> `amdgpu-dkms` replaces the in-tree `amdgpu` module of the same name; `depmod` -> prefers the DKMS copy. The script just autoloads `amdgpu`. +> the DKMS `amdgpu` module replaces the in-tree one via `depmod`'s `updates/` +> override. The script just autoloads `amdgpu`. + +In `inbox` mode steps 2–5 are skipped entirely; the script only ensures +`amdgpu` autoloads and rebuilds the initrd. --- @@ -88,7 +106,8 @@ It runs in the `base-image` target **after** the kernel is finalized. ARCH=amd64 INSTALL_AMD_GPU_DRIVERS=true - AMDGPU_ROCM_VERSION=7.2.4 + AMDGPU_DRIVER_SOURCE=dkms # or "inbox" — see modes above + AMDGPU_DRIVER_RELEASE=31.30 # amdgpu-install release marker (dkms mode only) ``` 2. Build as usual, e.g.: @@ -102,9 +121,13 @@ It runs in the `base-image` target **after** the kernel is finalized. ```sh ./earthly.sh +base-image --ARCH=amd64 \ --INSTALL_AMD_GPU_DRIVERS=true \ - --AMDGPU_ROCM_VERSION=6.4.4 + --AMDGPU_DRIVER_RELEASE=31.30 ``` + If the `dkms` build fails on your image kernel (see the mapping table + below), rebuild with `--AMDGPU_DRIVER_SOURCE=inbox` to fall back to the + in-tree driver instead. + 3. Mirror the AMD GPU Operator container images into your Palette content bundle and install the operator with `driver.enable=false`. @@ -114,15 +137,58 @@ It runs in the `base-image` target **after** the kernel is finalized. | Variable | Default | Description | | --- | --- | --- | -| `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Bakes `amdgpu-dkms` into the Ubuntu base image. | -| `AMDGPU_ROCM_VERSION` | `7.2.4` | ROCm/driver release to install. Selects the `amdgpu-install` package under `repo.radeon.com/amdgpu-install//`, which configures the matching driver repo. **Must match the ROCm version of the operator images you bundle.** | +| `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Enables the AMD pre-install pipeline. | +| `AMDGPU_DRIVER_SOURCE` | `dkms` | `dkms` (build AMD's out-of-tree driver against the image kernel) or `inbox` (skip the AMD repo and use the in-tree amdgpu). See modes above. | +| `AMDGPU_DRIVER_RELEASE` | `31.30` | **`dkms` mode only.** `amdgpu-install` release marker (URL segment under `repo.radeon.com/amdgpu-install//`, e.g. `31.30`, `31.10`, `30.30.4`). Note this is AMD's *driver-release* identifier, not a ROCm x.y.z version — the two schemes coexist and only a subset of ROCm aliases are published as URL paths. | | `AMDGPU_REBUILD_INITRD` | `true` | Rebuild the initrd for the image kernel. | -### Choosing a version +### Choosing a driver release (snapshot, 2026-07-11) + +The table below is a **snapshot** as of 2026-07-11 of what +`repo.radeon.com/amdgpu//ubuntu/dists/{jammy,noble}/…/Packages` +publishes for `amdgpu-dkms`. AMD adds new releases periodically — always +cross-check the authoritative matrix before pinning a value: -Browse available releases at - (e.g. `7.2.4`, `6.4.4`, `6.2.2`). Pick -the one matching your hardware and the operator images in your bundle. +- [AMD ROCm on Linux — system requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html) (canonical kernel matrix) +- Repo index: (published release markers) + +| `AMDGPU_DRIVER_RELEASE` | `amdgpu-dkms` source | Notes | +| --- | --- | --- | +| `30.30.4` | `6.16.13` | Paired with ROCm 7.2.4. Empirically **does not build against Linux 6.17** despite AMD docs listing it as supported for 24.04.4 HWE. Was CanvOS's default before this fix. | +| `31.10` | `6.18.4` | Newer source; picks up mainline through ~6.18. | +| `31.20` | `6.19.0` | | +| **`31.30` (current default)** | `6.19.4` | Confirmed builds against Linux 6.17 on Ubuntu 22.04 / 24.04. Publishes for jammy, noble, and resolute (25.04). | + +**How to pick:** the default `31.30` is fine for anyone on 22.04 or 24.04 in +mid-2026. Bump it only when a newer release is required to match the ROCm +user-space release of the operator images you bundle, or if AMD publishes a +release with fixes for your specific silicon. Check AMD's release notes for the +authoritative ROCm ↔ driver-release pairing; the URL scheme intentionally does +**not** encode ROCm x.y.z in the release marker. + +> **Docs vs reality:** AMD's system-requirements page has been observed to list +> kernel support that the shipped driver source doesn't yet build against +> (e.g. 30.30.4 with 6.17 on 24.04). Trust the empirical result: if the build +> fails, bump the release or fall back to `inbox`. + +### When the DKMS build fails + +The most common cause is a **kernel newer than what the chosen driver release +supports** (e.g. running Linux 6.17 with `AMDGPU_DRIVER_RELEASE=30.30.4`, +whose 6.16.13 source doesn't build against 6.17). The build will fail loudly +with the `make.log` tail and a pointer at this document. + +Two ways to unblock: + +1. **Bump `AMDGPU_DRIVER_RELEASE`** to a newer marker whose source supports + your kernel (see the table above). This keeps you on AMD's out-of-tree + driver. +2. **Rebuild with `AMDGPU_DRIVER_SOURCE=inbox`** if no supported release + exists yet, or if you're willing to trade the OOT driver's newer + SMU/per-SKU support for whatever the in-tree amdgpu ships with your kernel. + Enumeration and compute typically work on established silicon (e.g. MI300 + family) with the in-tree driver; expect the operator to lose the + `amd.com/gpu.driver-version` node label. --- @@ -132,6 +198,9 @@ the one matching your hardware and the operator images in your bundle. lsmod | grep amdgpu dmesg | grep -i amdgpu ls /sys/class/kfd 2>/dev/null && echo "KFD present" +cat /etc/canvos/amdgpu-driver-source # which mode ran + release info +# In dkms mode, expect a module under /lib/modules//updates/dkms/ +find /lib/modules/$(uname -r)/updates -name 'amdgpu.ko*' 2>/dev/null # If you also bundle ROCm user-space tooling: # rocminfo ; amd-smi list ``` @@ -196,5 +265,9 @@ operator pods reach `Ready`. - **Secure Boot / UKI is not supported by this path** (unsigned DKMS modules won't load). Use the standard (non-UKI) Ubuntu image for GPU nodes. - **amd64 / Ubuntu only.** -- **Version alignment is yours to own** — `AMDGPU_ROCM_VERSION` must line up with - the ROCm version of the operator images you bundle. +- **Version alignment is yours to own** — `AMDGPU_DRIVER_RELEASE` must line up + with the ROCm version of the operator images you bundle. +- **`inbox` mode loses the driver-version node label** — the AMD GPU Operator's + node-labeller reads `/sys/class/drm/card*/device/driver/module/version`, + which only exists when the driver was DKMS-installed. Enumeration and + scheduling still work; driver-version-aware policies won't. diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index 9b6bca10..46f685b4 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -2,29 +2,40 @@ # # install-amdgpu-drivers.sh # -# Pre-install the AMD Instinct GPU kernel-mode driver (amdgpu-dkms) INTO a -# CanvOS / Kairos Ubuntu base image, so a node booted from the image can run -# the AMD GPU Operator in a fully air-gapped environment WITHOUT any host-side -# network access and WITHOUT the operator building/managing the driver. +# Pre-provision the AMD Instinct GPU kernel-mode driver INTO a CanvOS / Kairos +# Ubuntu base image, so a node booted from the image can run the AMD GPU Operator +# with `driver.enable=false` in a fully air-gapped environment. # -# WHAT THIS COVERS (OS side only) -# ------------------------------- +# TWO MODES (AMDGPU_DRIVER_SOURCE) +# -------------------------------- +# dkms (default) -- download AMD's amdgpu-dkms source from repo.radeon.com and +# DKMS-build it against the image kernel. Recommended for +# Instinct/MI silicon, where the in-tree driver typically +# lags on SMU firmware interfaces and per-SKU features. +# inbox -- do NOT install the AMD apt repo or amdgpu-dkms. Rely on +# the in-tree `amdgpu` module that Ubuntu ships with +# linux-modules-$(uname -r) and the firmware blobs in +# linux-firmware. Only ensures amdgpu autoloads. Choose +# this when the DKMS build fails against your image kernel +# (e.g. AMD hasn't published a driver release supporting +# your kernel yet) and you accept the in-tree driver's +# feature set (may miss recent SMU / per-SKU support). +# +# WHAT THIS COVERS (dkms mode, OS side only) +# ------------------------------------------ # * build toolchain (gcc, make, dkms, kmod, libc headers) -# * kernel headers that match the kernel shipped in the image -# (delegated to scripts/install-kernel-headers.sh) -# * linux-modules-extra for the image kernel (amdgpu pulls modules from it) -# * the AMD amdgpu kernel module built with DKMS against the IMAGE kernel -# (amdgpu-dkms + amdgpu-dkms-firmware) +# * kernel headers matching the image kernel (via install-kernel-headers.sh) +# * linux-modules-extra for the image kernel +# * amdgpu-dkms + amdgpu-dkms-firmware, built against the IMAGE kernel # * amdgpu module autoload + initrd refresh +# * a marker at /etc/canvos/amdgpu-driver-source recording which mode ran # -# WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, -# deployed by the AMD GPU Operator itself): +# WHAT THIS DOES *NOT* COVER (both modes) -- ship these as container images in +# your content bundle, deployed by the AMD GPU Operator itself: # * ROCm user-space, device-plugin, node-labeller, metrics exporter, etc. # # At Helm-install time you MUST tell the operator the driver is pre-installed: # --set driver.enable=false # note: "enable", not "enabled" -# (the operator then "directly uses inbox or pre-installed AMD GPU drivers" -# and only deploys device-plugin / node-labeller / metrics-exporter) # # WHY THE DKMS DANCE (same rationale as install-nvidia-drivers.sh) # --------------------------------------------------------------- @@ -33,28 +44,17 @@ # kernel that will actually boot) and force the DKMS build + module install + # depmod against THAT kernel. # -# NOTE ON BLACKLISTING -# -------------------- -# Unlike NVIDIA (where the open-source `nouveau` driver must be blacklisted), -# amdgpu-dkms REPLACES the in-tree `amdgpu` module (same module name). depmod -# prefers the updates/dkms copy, so there is nothing to blacklist -- we simply -# ensure amdgpu autoloads. -# -# CONNECTIVITY -# ------------ -# This script runs at BUILD time, where the builder has internet. It bakes -# everything into the image. The resulting image needs no network at boot. -# # TUNABLES (environment variables; all optional) -# AMDGPU_ROCM_VERSION ROCm/driver release to install (e.g. 7.2.4, 6.4.4, -# 6.2.2). Default: 7.2.4 -# Must match the ROCm version of the operator images -# you bundle. This selects the amdgpu-install package -# under repo.radeon.com/amdgpu-install//, -# which configures the matching driver apt repo. +# AMDGPU_DRIVER_SOURCE dkms | inbox. Default: dkms. +# AMDGPU_DRIVER_RELEASE amdgpu-install release marker (URL segment under +# repo.radeon.com/amdgpu-install//), e.g. 31.30, +# 31.10, 30.30.4. Default: 31.30. Ignored in inbox +# mode. See docs/amd-gpu-airgapped.md for a snapshot +# of release-to-kernel compatibility. # AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. -# Default: true +# Default: true. # +set -eo pipefail set -u log() { echo "[install-amdgpu-drivers] $*"; } @@ -64,9 +64,15 @@ die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- -AMDGPU_ROCM_VERSION="${AMDGPU_ROCM_VERSION:-7.2.4}" +AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.30}" AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-true}" +case "${AMDGPU_DRIVER_SOURCE}" in + dkms|inbox) ;; + *) die "AMDGPU_DRIVER_SOURCE must be 'dkms' or 'inbox' (got: '${AMDGPU_DRIVER_SOURCE}')." ;; +esac + export DEBIAN_FRONTEND=noninteractive command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." @@ -77,7 +83,8 @@ command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based ( KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" [ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." log "Target (image) kernel: ${KVER}" -log "AMD ROCm/driver version: ${AMDGPU_ROCM_VERSION}" +log "Driver source mode: ${AMDGPU_DRIVER_SOURCE}" +[ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && log "AMD driver release: ${AMDGPU_DRIVER_RELEASE}" # Ubuntu release codename (jammy / noble) read from the image itself. codename="" @@ -90,6 +97,55 @@ if [ -r /etc/os-release ]; then fi [ -n "${codename}" ] || die "could not determine Ubuntu codename from /etc/os-release." +mkdir -p /etc/canvos + +# --------------------------------------------------------------------------- +# INBOX MODE: skip the AMD apt repo and DKMS entirely. Rely on the in-tree +# amdgpu module shipped with the image's linux-modules-* package. Only ensure +# the module autoloads at boot, then rebuild initrd if requested. +# --------------------------------------------------------------------------- +if [ "${AMDGPU_DRIVER_SOURCE}" = "inbox" ]; then + log "inbox mode: verifying in-tree amdgpu module is present under /lib/modules/${KVER}/kernel/..." + if ! find "/lib/modules/${KVER}" -path '*/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko*' 2>/dev/null | grep -q .; then + die "inbox mode selected but no in-tree amdgpu module found under /lib/modules/${KVER}/kernel/. \ +This image kernel does not ship an in-tree amdgpu driver -- switch to \ +AMDGPU_DRIVER_SOURCE=dkms or pick a different image kernel." + fi + + log "Configuring amdgpu module autoload ..." + cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh (inbox mode) +# Load the in-tree AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + printf 'AMDGPU_DRIVER_SOURCE=inbox\nKVER=%s\n' "${KVER}" > /etc/canvos/amdgpu-driver-source + + log "Done. Using in-tree amdgpu driver for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE begins here. +# --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # 2. Build toolchain # --------------------------------------------------------------------------- @@ -141,17 +197,18 @@ fi # --------------------------------------------------------------------------- # 4. Register the AMD driver repo via the amdgpu-install package # The amdgpu-install .deb (AMD's blessed entry point) configures the correct -# versioned driver apt repo + GPG key for the requested ROCm release. Its +# versioned driver apt repo + GPG key for the requested release. Its # filename carries a build number, so we auto-discover it from the directory # listing rather than hardcoding it. # --------------------------------------------------------------------------- -inst_dir="https://repo.radeon.com/amdgpu-install/${AMDGPU_ROCM_VERSION}/${osid}/${codename}" +inst_dir="https://repo.radeon.com/amdgpu-install/${AMDGPU_DRIVER_RELEASE}/${osid}/${codename}" log "Locating amdgpu-install package under ${inst_dir}/ ..." deb_name="$(curl -fsSL "${inst_dir}/" 2>/dev/null \ | grep -oE 'amdgpu-install_[0-9A-Za-z._-]+_all\.deb' | sort -u | tail -1)" -[ -n "${deb_name}" ] || die "could not find an amdgpu-install package for ROCm \ -${AMDGPU_ROCM_VERSION} / ${codename} at ${inst_dir}/. \ -Check available versions at https://repo.radeon.com/amdgpu-install/" +[ -n "${deb_name}" ] || die "could not find an amdgpu-install package for driver \ +release '${AMDGPU_DRIVER_RELEASE}' on ${codename} at ${inst_dir}/. Check available \ +releases at https://repo.radeon.com/amdgpu-install/ and pick one that supports \ +your image kernel (${KVER}); see docs/amd-gpu-airgapped.md for the mapping." log "Installing ${deb_name} (configures the AMD driver apt repo) ..." wget -qO /tmp/amdgpu-install.deb "${inst_dir}/${deb_name}" \ @@ -161,72 +218,100 @@ rm -f /tmp/amdgpu-install.deb apt-get update || warn "apt-get update after adding the AMD repo failed." # --------------------------------------------------------------------------- -# 5. Install the kernel-mode driver only (amdgpu-dkms + firmware) +# 5. Install the kernel-mode driver (amdgpu-dkms + firmware). +# Any apt/postinst failure surfaces here -- DO NOT swallow errors; a broken +# DKMS build must fail the image build so the user can fix AMDGPU_DRIVER_RELEASE +# or fall back to AMDGPU_DRIVER_SOURCE=inbox. # --------------------------------------------------------------------------- -log "Installing amdgpu-dkms ..." -# Note: amdgpu-dkms post-install script may fail in container environments -# due to missing EFI support. We attempt the install and continue even if dpkg -# post-install fails, then manually fix the configuration. -apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware 2>&1 | grep -v "dpkg: error" || true -apt-get install -y amdgpu-dkms 2>&1 | grep -v "dpkg: error" || true - -# Force-configure any packages with broken post-install scripts -log "Force-configuring packages with broken installations..." -dpkg --configure -a --force-all 2>&1 || true - -# Verify amdgpu-dkms was at least partially installed -if ! dpkg -l | grep -q "amdgpu-dkms"; then - die "failed to install amdgpu-dkms. \ -List available driver packages with: apt-cache search amdgpu-dkms" +log "Installing amdgpu-dkms + amdgpu-dkms-firmware ..." +if ! apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware; then + die "failed to install amdgpu-dkms (release '${AMDGPU_DRIVER_RELEASE}') for \ +kernel ${KVER}. Inspect /var/lib/dkms/amdgpu/*/build/make.log inside the failed \ +build layer. Common causes: (1) the AMD driver source in this release does not \ +support this kernel -- bump AMDGPU_DRIVER_RELEASE to a newer marker (see \ +docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed. \ +As a workaround, rerun with AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." fi -log "amdgpu-dkms package installation completed (post-install script errors suppressed)." - # --------------------------------------------------------------------------- -# 6. Build the DKMS module against the IMAGE kernel (not the build host) +# 6. Build the DKMS module against the IMAGE kernel (not the build host). +# apt-get's postinst may have already tried against $KVER; we re-run +# explicitly and let failures propagate (no `|| true`). # --------------------------------------------------------------------------- -if command -v dkms >/dev/null 2>&1; then - log "Building amdgpu DKMS module for kernel ${KVER} ..." - # `dkms status` differs across versions: - # dkms 2.x: "amdgpu, 6.16.13, 6.14.0-36-generic, x86_64: installed" - # dkms 3.x: "amdgpu/6.16.13, 6.14.0-36-generic, x86_64: installed" - # Extract module name (up to first , / or :) + first version-looking token. - dkms status 2>/dev/null | grep -i amdgpu | while read -r line; do - mod="$(printf '%s\n' "${line}" | sed -E 's/[,/:].*//' | tr -d ' ')" - ver="$(printf '%s\n' "${line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" - case "${mod}" in amdgpu*) ;; *) continue ;; esac - [ -n "${mod}" ] && [ -n "${ver}" ] || continue - log " dkms install ${mod}/${ver} -k ${KVER}" - dkms build -m "${mod}" -v "${ver}" -k "${KVER}" 2>/dev/null || true - dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force 2>/dev/null || true - done - # Belt-and-suspenders: autoinstaller pinned to the target kernel. - dkms autoinstall -k "${KVER}" 2>/dev/null || true - log "DKMS status:"; dkms status 2>/dev/null || true -else - warn "dkms not found; relying on apt postinst build." +command -v dkms >/dev/null 2>&1 || die "dkms binary not found after installing amdgpu-dkms." + +log "Building amdgpu DKMS module for kernel ${KVER} ..." +# `dkms status` differs across versions: +# dkms 2.x: "amdgpu, 6.19.4, 6.14.0-36-generic, x86_64: installed" +# dkms 3.x: "amdgpu/6.19.4, 6.14.0-36-generic, x86_64: installed" +# We want the module-name and source-version (not the kernel). +dkms_line="$(dkms status 2>/dev/null | grep -iE '^amdgpu[/,]' | head -1 || true)" +[ -n "${dkms_line}" ] || die "dkms status does not know about the amdgpu module \ +after apt install -- driver package is broken or DKMS registration failed." + +mod="$(printf '%s\n' "${dkms_line}" | sed -E 's/[,/:].*//' | tr -d ' ')" +ver="$(printf '%s\n' "${dkms_line}" | sed -E 's/^[^/,]+[/,] *//' | sed -E 's/[,:].*//' | tr -d ' ')" +# Fallback: grep any version-looking token if the second column wasn't the version. +if ! printf '%s' "${ver}" | grep -qE '^[0-9]+\.[0-9]+'; then + ver="$(printf '%s\n' "${dkms_line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" +fi +[ -n "${mod}" ] && [ -n "${ver}" ] || die "could not parse dkms status line: '${dkms_line}'" + +log " dkms build ${mod}/${ver} -k ${KVER}" +if ! dkms build -m "${mod}" -v "${ver}" -k "${KVER}"; then + log "DKMS build failed. Full make.log tail:" + tail -n 60 "/var/lib/dkms/${mod}/${ver}/build/make.log" 2>&1 | sed 's/^/ /' || true + die "DKMS build of ${mod}/${ver} against kernel ${KVER} failed. AMD driver \ +release '${AMDGPU_DRIVER_RELEASE}' likely does not support this kernel. Either \ +bump AMDGPU_DRIVER_RELEASE (see docs/amd-gpu-airgapped.md) or rerun with \ +AMDGPU_DRIVER_SOURCE=inbox." fi +log " dkms install ${mod}/${ver} -k ${KVER}" +dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force \ + || die "dkms install of ${mod}/${ver} against kernel ${KVER} failed." + +log "DKMS status:"; dkms status 2>&1 | sed 's/^/ /' || true + # --------------------------------------------------------------------------- -# 7. Verify the module landed in the image kernel's module tree +# 7. Verify the DKMS-built module actually landed under updates/dkms and that +# dkms considers it installed for the target kernel. The in-tree amdgpu +# that Ubuntu ships under kernel/... does NOT count -- we're only satisfied +# if the OOT driver made it in. # --------------------------------------------------------------------------- MODDIR="/lib/modules/${KVER}" -if find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then - log "Verified: amdgpu kernel module present under ${MODDIR}." - find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' -else - die "no amdgpu.ko module found under ${MODDIR} -- DKMS build did not produce \ -a module for the image kernel. Check that linux-headers-${KVER} and a matching \ -gcc are installed." +dkms_mod_found="" +if find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + dkms_mod_found="yes" +fi + +dkms_installed="" +if dkms status 2>/dev/null \ + | grep -iE "^amdgpu[/,][^,]*,[[:space:]]*${KVER}[,]" \ + | grep -q ': installed'; then + dkms_installed="yes" +fi + +if [ -z "${dkms_mod_found}" ] || [ -z "${dkms_installed}" ]; then + log "Verification failed:" + log " updates/dkms module present under ${MODDIR}/updates: ${dkms_mod_found:-no}" + log " dkms status shows 'installed' for kernel ${KVER}: ${dkms_installed:-no}" + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' || true + die "amdgpu DKMS module was NOT built+installed for kernel ${KVER}. The \ +in-tree amdgpu (if any) is NOT sufficient in dkms mode -- rerun with \ +AMDGPU_DRIVER_SOURCE=inbox if that is what you want." fi +log "Verified: amdgpu DKMS module installed for ${KVER}." +find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' + # --------------------------------------------------------------------------- # 8. Autoload amdgpu at boot (no blacklist needed -- dkms replaces the in-tree -# module of the same name) +# module of the same name via depmod's updates/ override). # --------------------------------------------------------------------------- log "Configuring amdgpu module autoload ..." cat > /etc/modules-load.d/amdgpu.conf <<'EOF' -# Managed by CanvOS install-amdgpu-drivers.sh +# Managed by CanvOS install-amdgpu-drivers.sh (dkms mode) # Load the AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. amdgpu EOF @@ -253,10 +338,17 @@ elif [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/de fi # --------------------------------------------------------------------------- -# 11. Cleanup apt caches to keep the image lean +# 11. Record what we did so ops can query it on-node. +# --------------------------------------------------------------------------- +printf 'AMDGPU_DRIVER_SOURCE=dkms\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_DKMS_MODULE=%s/%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${mod}" "${ver}" "${KVER}" \ + > /etc/canvos/amdgpu-driver-source + +# --------------------------------------------------------------------------- +# 12. Cleanup apt caches to keep the image lean # --------------------------------------------------------------------------- apt-get clean rm -rf /var/lib/apt/lists/* -log "Done. AMD amdgpu driver (ROCm ${AMDGPU_ROCM_VERSION}) baked in for kernel ${KVER}." +log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}) baked in for kernel ${KVER}." log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." From ad4c6fa536aa0ecc2037fed1e0000c1d2d909a17 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 13:04:00 -0700 Subject: [PATCH 11/34] fix(gpu): skip DKMS module signing in AMD driver install (no UEFI in build container) amdgpu-dkms >= 6.18 (observed on release 31.30 = amdgpu-dkms 6.19.4) invokes mokutil from the DKMS sign_tool hook to enroll a Machine Owner Key. mokutil reads /sys/firmware/efi/efivars, which is not exposed inside Docker/Earthly RUN containers, so it aborts with: "EFI variables are not supported on this system / /sys/firmware/efi/efivars not found, aborting." and the amdgpu-dkms postinst returns non-zero, failing the base-image build. Drop /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf with sign_tool="" immediately before apt-get install amdgpu-dkms. DKMS honors this and skips signing entirely, sidestepping the mokutil invocation. Modules are unsigned; this path already documents that Secure Boot / UKI is unsupported. The step-5 error message now lists this as the third common failure mode so future regressions surface with a pointer at the drop-in. Co-Authored-By: Claude Opus 4.7 --- scripts/install-amdgpu-drivers.sh | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index 46f685b4..bbbc5bab 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -217,6 +217,31 @@ apt-get install -y /tmp/amdgpu-install.deb || die "failed to install amdgpu-inst rm -f /tmp/amdgpu-install.deb apt-get update || warn "apt-get update after adding the AMD repo failed." +# --------------------------------------------------------------------------- +# 4b. Disable DKMS module signing before installing amdgpu-dkms. +# +# amdgpu-dkms (>= 6.18 range, and observed on 31.x releases) invokes mokutil +# from the DKMS sign_tool hook to enroll a Machine Owner Key, which reads +# /sys/firmware/efi/efivars. Docker/Earthly build containers don't expose +# efivars, so mokutil aborts with: +# "EFI variables are not supported on this system / +# /sys/firmware/efi/efivars not found, aborting." +# and the amdgpu-dkms postinst returns non-zero. Empty sign_tool tells DKMS +# to skip signing entirely, sidestepping the mokutil invocation. +# +# CAVEAT: modules produced this way are unsigned -- consistent with the +# Secure Boot / UKI limitation already documented in docs/amd-gpu-airgapped.md. +# --------------------------------------------------------------------------- +log "Disabling DKMS module signing (container has no UEFI efivars) ..." +mkdir -p /etc/dkms/framework.conf.d +cat > /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Empty sign_tool tells DKMS to skip module signing. Required for building +# amdgpu-dkms inside container image builds where /sys/firmware/efi/efivars +# is not available. Modules are unsigned; this path does not support Secure Boot. +sign_tool="" +EOF + # --------------------------------------------------------------------------- # 5. Install the kernel-mode driver (amdgpu-dkms + firmware). # Any apt/postinst failure surfaces here -- DO NOT swallow errors; a broken @@ -229,8 +254,10 @@ if ! apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware kernel ${KVER}. Inspect /var/lib/dkms/amdgpu/*/build/make.log inside the failed \ build layer. Common causes: (1) the AMD driver source in this release does not \ support this kernel -- bump AMDGPU_DRIVER_RELEASE to a newer marker (see \ -docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed. \ -As a workaround, rerun with AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." +docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed; \ +(3) DKMS module signing failed reaching /sys/firmware/efi/efivars -- normally \ +handled by the sign_tool='' drop-in above; check it exists and is readable. \ +Workaround: rerun with AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." fi # --------------------------------------------------------------------------- From d43c74a9a6b5f44bfe29da808b221778aa0c6fce Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 13:30:50 -0700 Subject: [PATCH 12/34] chore(gpu): default AMDGPU_DRIVER_RELEASE to 7.2.1 (matches GPU Operator v1.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empirical reproduction on the kairos-ubuntu:24.04-core-amd64-generic-v4.0.4 base image (kernel 6.17.0-35-generic) with the AMDGPU_ROCM_VERSION=7.2.4-era script + Abhijit's masking reverted showed that amdgpu-dkms 6.16.13 builds cleanly against Linux 6.17: DKMS reports the module `installed`, all 8 amd*.ko files land under /lib/modules//updates/dkms/, and apt-get returns 0. The "EFI variables are not supported on this system / /sys/firmware/efi/efivars not found, aborting" line printed during the 30.30.x postinst is a cosmetic mokutil warning (sign_tool step); the postinst continues past it and finishes normally. The signing config drop-in from ad4c6fa is still valuable as defensive belt-and-suspenders for the 31.x line where the same message is treated as fatal. Given that, revert the previous 31.30 default (which crossed AMD's tech- preview boundary) back to 7.2.1 -- the release AMD's GPU Operator v1.5.0 release notes explicitly pair the operator with. AMD publishes both ROCm-alias and driver-release-marker URL segments at repo.radeon.com/amdgpu-install/, so "7.2.1" is a valid AMDGPU_DRIVER_RELEASE (equivalent to "30.30.1"). Also refactor the docs' "Version alignment" section to lead with the operator → ROCm → driver → kernel chain and explicitly label the 31.x line as tech-preview. Co-Authored-By: Claude Opus 4.7 --- .arg.template | 13 ++-- Earthfile | 9 ++- docs/amd-gpu-airgapped.md | 116 ++++++++++++++++++------------ scripts/install-amdgpu-drivers.sh | 15 ++-- 4 files changed, 96 insertions(+), 57 deletions(-) diff --git a/.arg.template b/.arg.template index a89a2e9b..3c44af61 100644 --- a/.arg.template +++ b/.arg.template @@ -56,9 +56,12 @@ FORCE_INTERACTIVE_INSTALL=false # # and skips the AMD apt repo entirely — use # # when the DKMS build fails against your image # # kernel (see docs). -# AMDGPU_DRIVER_RELEASE=31.30 # Only used with dkms mode. amdgpu-install -# # release marker (URL segment under -# # repo.radeon.com/amdgpu-install//, e.g. -# # 31.30, 31.10, 30.30.4). See the version- -# # pick table in docs/amd-gpu-airgapped.md. +# AMDGPU_DRIVER_RELEASE=7.2.1 # Only used with dkms mode. amdgpu-install +# # release marker. Default 7.2.1 pairs with +# # GPU Operator v1.5.0 (ROCm 7.2.1). AMD also +# # publishes driver-release-marker paths like +# # 30.30.1 / 30.30.4 / 31.30; either form works. +# # 31.x is tech-preview -- do not mix with a +# # production operator. See the version- +# # alignment table in docs/amd-gpu-airgapped.md. # AMDGPU_REBUILD_INITRD=true # Rebuild initrd for the image kernel diff --git a/Earthfile b/Earthfile index 5197bb02..32903267 100644 --- a/Earthfile +++ b/Earthfile @@ -83,9 +83,12 @@ ARG INSTALL_AMD_GPU_DRIVERS=false # fails against your image kernel and you accept the in-tree driver's feature set. ARG AMDGPU_DRIVER_SOURCE=dkms # amdgpu-install release marker (URL segment under repo.radeon.com/amdgpu-install//). -# Default 31.30 ships amdgpu-dkms 6.19.4, which builds against Linux kernels through -# 6.17 (Ubuntu 22.04 & 24.04 HWE range). See docs/amd-gpu-airgapped.md for the mapping. -ARG AMDGPU_DRIVER_RELEASE=31.30 +# Default 7.2.1 pairs with AMD GPU Operator v1.5.0 (per its release notes) and installs +# amdgpu-dkms 6.16.13 (30.30.1 line). Empirically builds cleanly against Linux kernels +# through 6.17 on Ubuntu 24.04. AMD publishes both ROCm-alias (7.2.1) and driver-release- +# marker (30.30.1, 31.30) URL segments; either form is accepted here. The 31.x line is +# tech-preview -- do not mix with a production operator. See docs/amd-gpu-airgapped.md. +ARG AMDGPU_DRIVER_RELEASE=7.2.1 ARG AMDGPU_REBUILD_INITRD=true # NVIDIA and AMD driver pre-install are mutually exclusive within a single image. diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md index 67c01383..ab2ef767 100644 --- a/docs/amd-gpu-airgapped.md +++ b/docs/amd-gpu-airgapped.md @@ -107,7 +107,7 @@ In `inbox` mode steps 2–5 are skipped entirely; the script only ensures INSTALL_AMD_GPU_DRIVERS=true AMDGPU_DRIVER_SOURCE=dkms # or "inbox" — see modes above - AMDGPU_DRIVER_RELEASE=31.30 # amdgpu-install release marker (dkms mode only) + AMDGPU_DRIVER_RELEASE=7.2.1 # pairs with GPU Operator v1.5.0 (dkms mode only) ``` 2. Build as usual, e.g.: @@ -121,7 +121,7 @@ In `inbox` mode steps 2–5 are skipped entirely; the script only ensures ```sh ./earthly.sh +base-image --ARCH=amd64 \ --INSTALL_AMD_GPU_DRIVERS=true \ - --AMDGPU_DRIVER_RELEASE=31.30 + --AMDGPU_DRIVER_RELEASE=7.2.1 ``` If the `dkms` build fails on your image kernel (see the mapping table @@ -139,56 +139,84 @@ In `inbox` mode steps 2–5 are skipped entirely; the script only ensures | --- | --- | --- | | `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Enables the AMD pre-install pipeline. | | `AMDGPU_DRIVER_SOURCE` | `dkms` | `dkms` (build AMD's out-of-tree driver against the image kernel) or `inbox` (skip the AMD repo and use the in-tree amdgpu). See modes above. | -| `AMDGPU_DRIVER_RELEASE` | `31.30` | **`dkms` mode only.** `amdgpu-install` release marker (URL segment under `repo.radeon.com/amdgpu-install//`, e.g. `31.30`, `31.10`, `30.30.4`). Note this is AMD's *driver-release* identifier, not a ROCm x.y.z version — the two schemes coexist and only a subset of ROCm aliases are published as URL paths. | +| `AMDGPU_DRIVER_RELEASE` | `7.2.1` | **`dkms` mode only.** `amdgpu-install` URL segment under `repo.radeon.com/amdgpu-install//`. AMD publishes both ROCm-alias paths (e.g. `7.2.1`, `7.2.4`) and driver-release-marker paths (e.g. `30.30.1`, `30.30.4`, `31.30`) — either form works. Default `7.2.1` pairs with **GPU Operator v1.5.0** (per AMD's release notes) → **ROCm 7.2.1** → **amdgpu-dkms 6.16.13** (30.30.1 build). | | `AMDGPU_REBUILD_INITRD` | `true` | Rebuild the initrd for the image kernel. | -### Choosing a driver release (snapshot, 2026-07-11) +### Version alignment across the stack (snapshot, 2026-07-11) -The table below is a **snapshot** as of 2026-07-11 of what -`repo.radeon.com/amdgpu//ubuntu/dists/{jammy,noble}/…/Packages` -publishes for `amdgpu-dkms`. AMD adds new releases periodically — always -cross-check the authoritative matrix before pinning a value: +Five things have to line up to have a supportable node. Start from the operator +version you bundle and follow AMD's release notes / compat matrix from there: -- [AMD ROCm on Linux — system requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html) (canonical kernel matrix) -- Repo index: (published release markers) +``` + GPU Operator ─┐ AMD release notes pair the operator with a specific + │ ROCm user-space release + ROCm user-space (device-plugin / metrics-exporter / etc) + │ AMD user↔kernel compat matrix pairs ROCm with a + │ driver-release marker + amdgpu driver (amdgpu-dkms) → this is what this script installs + │ The DKMS source has a supported kernel window + Image kernel ─┘ + Kubernetes version — validated per operator release +``` -| `AMDGPU_DRIVER_RELEASE` | `amdgpu-dkms` source | Notes | -| --- | --- | --- | -| `30.30.4` | `6.16.13` | Paired with ROCm 7.2.4. Empirically **does not build against Linux 6.17** despite AMD docs listing it as supported for 24.04.4 HWE. Was CanvOS's default before this fix. | -| `31.10` | `6.18.4` | Newer source; picks up mainline through ~6.18. | -| `31.20` | `6.19.0` | | -| **`31.30` (current default)** | `6.19.4` | Confirmed builds against Linux 6.17 on Ubuntu 22.04 / 24.04. Publishes for jammy, noble, and resolute (25.04). | - -**How to pick:** the default `31.30` is fine for anyone on 22.04 or 24.04 in -mid-2026. Bump it only when a newer release is required to match the ROCm -user-space release of the operator images you bundle, or if AMD publishes a -release with fixes for your specific silicon. Check AMD's release notes for the -authoritative ROCm ↔ driver-release pairing; the URL scheme intentionally does -**not** encode ROCm x.y.z in the release marker. - -> **Docs vs reality:** AMD's system-requirements page has been observed to list -> kernel support that the shipped driver source doesn't yet build against -> (e.g. 30.30.4 with 6.17 on 24.04). Trust the empirical result: if the build -> fails, bump the release or fall back to `inbox`. +**Two parallel driver tracks** — do not mix them: + +| Track | `AMDGPU_DRIVER_RELEASE` values | ROCm user-space | Paired GPU Operator | +| --- | --- | --- | --- | +| **Production** | `7.2.1` (= `30.30.1`), `7.2.4` (= `30.30.4`), etc. | ROCm 7.2.x | **v1.5.0 (what CanvOS bundles)** | +| Tech preview | `31.10` / `31.20` / `31.30` | ROCm 7.13.0 tech-preview | not yet paired with a released operator | + +Authoritative references: +- [AMD GPU Operator v1.5.0 release notes](https://instinct.docs.amd.com/projects/gpu-operator/en/main/releasenotes.html#gpu-operator-v1-5-0-release-notes) +- [ROCm user↔kernel compat matrix](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html) +- [ROCm on Linux system requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html) +- Repo index: + +Snapshot of what `repo.radeon.com/amdgpu//ubuntu/dists/noble/…/Packages` +publishes for `amdgpu-dkms`: + +| `AMDGPU_DRIVER_RELEASE` | `amdgpu-dkms` build | Track | Paired with | +| --- | --- | --- | --- | +| **`7.2.1` (= `30.30.1`, default)** | `6.16.13-2303411` | Production | ROCm 7.2.1 → GPU Operator v1.5.0 | +| `7.2.4` (= `30.30.4`) | `6.16.13-2341068` | Production | ROCm 7.2.4 | +| `31.10` | `6.18.4` | Tech preview | ROCm 7.13.0 tech-preview | +| `31.30` | `6.19.4` | Tech preview | ROCm 7.13.0 tech-preview | + +The whole 30.30.x line uses the same driver *source* (`6.16.13`); only the build +number and paired user-space differ. Empirically, this source **builds cleanly +against Ubuntu 24.04's edge kernel 6.17** — the "EFI variables are not supported +on this system" line printed during postinst is a cosmetic mokutil warning +(sign_tool step); DKMS proceeds and lands the module under `updates/dkms/`. + +**How to pick:** default to the release marker paired with the operator you're +bundling. Bump only when you're also moving the operator to a paired version. +Do not switch to `31.x` for kernel-newness alone — that crosses into tech +preview and won't be validated with a production operator. ### When the DKMS build fails -The most common cause is a **kernel newer than what the chosen driver release -supports** (e.g. running Linux 6.17 with `AMDGPU_DRIVER_RELEASE=30.30.4`, -whose 6.16.13 source doesn't build against 6.17). The build will fail loudly -with the `make.log` tail and a pointer at this document. - -Two ways to unblock: - -1. **Bump `AMDGPU_DRIVER_RELEASE`** to a newer marker whose source supports - your kernel (see the table above). This keeps you on AMD's out-of-tree - driver. -2. **Rebuild with `AMDGPU_DRIVER_SOURCE=inbox`** if no supported release - exists yet, or if you're willing to trade the OOT driver's newer - SMU/per-SKU support for whatever the in-tree amdgpu ships with your kernel. - Enumeration and compute typically work on established silicon (e.g. MI300 - family) with the in-tree driver; expect the operator to lose the - `amd.com/gpu.driver-version` node label. +Common causes: + +1. **Kernel outside the driver's supported window** — `make.log` shows + `configure: cannot detect CFLAGS` or unresolved kernel symbols. Prefer + moving the image kernel into range (or the operator/ROCm/driver combo up + as a set) over jumping to a tech-preview driver. +2. **`linux-headers-` not installed for the image kernel** — check the + earlier log lines from `install-kernel-headers.sh`. Fix the headers. +3. **DKMS module signing (mokutil) failure in the container** — surfaces as + "EFI variables are not supported on this system / /sys/firmware/efi/efivars + not found, aborting." The script writes + `/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf` (empty `sign_tool`) + before the apt install to sidestep this. In 30.30.x the AMD postinst + already tolerates the missing EFI vars (it prints the warning and + continues); the sign_tool drop-in is defensive belt-and-suspenders for + 31.x and future releases that may treat it as fatal. + +The script prints the last 60 lines of `make.log` on failure. Read it — +the class of failure matters for the fix. Workaround for any of the above: +rerun with `AMDGPU_DRIVER_SOURCE=inbox` to use the in-tree amdgpu (accepts +the caveats above about missing driver-version label + SMU IF mismatch on +newer silicon). --- diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index bbbc5bab..53130716 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -47,10 +47,15 @@ # TUNABLES (environment variables; all optional) # AMDGPU_DRIVER_SOURCE dkms | inbox. Default: dkms. # AMDGPU_DRIVER_RELEASE amdgpu-install release marker (URL segment under -# repo.radeon.com/amdgpu-install//), e.g. 31.30, -# 31.10, 30.30.4. Default: 31.30. Ignored in inbox -# mode. See docs/amd-gpu-airgapped.md for a snapshot -# of release-to-kernel compatibility. +# repo.radeon.com/amdgpu-install//). AMD publishes +# both ROCm-alias paths (7.2.1, 7.2.4) and driver- +# release-marker paths (30.30.1, 30.30.4, 31.30); +# either form is accepted. Default: 7.2.1 -- pairs +# with GPU Operator v1.5.0 per its release notes. +# The 31.x line is tech-preview and pairs only with +# ROCm 7.13.0 tech-preview; do not mix with a +# production operator. Ignored in inbox mode. See +# docs/amd-gpu-airgapped.md for the compat matrix. # AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. # Default: true. # @@ -65,7 +70,7 @@ die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } # Config # --------------------------------------------------------------------------- AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" -AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.30}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-true}" case "${AMDGPU_DRIVER_SOURCE}" in From 87efc43022bb74d684edb070cd3c43a7f93dd786 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 14:20:38 -0700 Subject: [PATCH 13/34] diag(gpu): dump DKMS make.log + env probes on amdgpu-dkms install failure Earthly's -i interactive debugger frequently hands out a broken tty in the buildkit sandbox, making /var/lib/dkms/*/build/make.log unreachable when amdgpu-dkms's postinst fails. Instead of asking users to debug interactively, tail the make.log(s) and a few environment probes (kernel headers state, Module.symvers presence, sign_tool drop-in state, MemAvailable) to stderr before die-ing. The next non-interactive rebuild surfaces the underlying compile error directly in the Earthly output. No behavior change on the happy path. Co-Authored-By: Claude Opus 4.7 --- scripts/install-amdgpu-drivers.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index 53130716..d89c3b9f 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -255,14 +255,30 @@ EOF # --------------------------------------------------------------------------- log "Installing amdgpu-dkms + amdgpu-dkms-firmware ..." if ! apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware; then + # apt/postinst failure -- dump the DKMS build artifacts so root-causing + # doesn't require an interactive session (Earthly's -i tty is often broken). + log "apt install failed. Dumping DKMS build artifacts for diagnosis:" + log "--- dkms status ---" + dkms status 2>&1 | sed 's/^/ /' || true + for f in /var/lib/dkms/amdgpu/*/build/make.log; do + [ -r "$f" ] || continue + log "--- ${f} (tail -150) ---" + tail -n 150 "$f" | sed 's/^/ /' || true + done + log "--- environment probes ---" + log " kernel: $(uname -r); target KVER: ${KVER}" + log " linux-headers pkg: $(dpkg -l "linux-headers-${KVER}" 2>/dev/null | awk '/^ii/{print $2, $3}')" + log " /lib/modules/${KVER}/build: $(readlink -f "/lib/modules/${KVER}/build" 2>/dev/null || echo MISSING)" + log " /usr/src/linux-headers-${KVER}/Module.symvers: $(test -s "/usr/src/linux-headers-${KVER}/Module.symvers" && echo present || echo missing/empty)" + log " sign_tool drop-in: $(test -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf && grep -E '^sign_tool' /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf || echo MISSING)" + log " memory: $(awk '/MemAvailable/{print $2/1024" MiB avail"}' /proc/meminfo)" die "failed to install amdgpu-dkms (release '${AMDGPU_DRIVER_RELEASE}') for \ -kernel ${KVER}. Inspect /var/lib/dkms/amdgpu/*/build/make.log inside the failed \ -build layer. Common causes: (1) the AMD driver source in this release does not \ -support this kernel -- bump AMDGPU_DRIVER_RELEASE to a newer marker (see \ -docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed; \ -(3) DKMS module signing failed reaching /sys/firmware/efi/efivars -- normally \ -handled by the sign_tool='' drop-in above; check it exists and is readable. \ -Workaround: rerun with AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." +kernel ${KVER}. See make.log tail above. Common causes: (1) the AMD driver source \ +in this release does not support this kernel -- bump AMDGPU_DRIVER_RELEASE (see \ +docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed / Module.symvers \ +empty; (3) DKMS module signing failed reaching /sys/firmware/efi/efivars -- normally \ +handled by the sign_tool='' drop-in above. Workaround: rerun with \ +AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." fi # --------------------------------------------------------------------------- From 1b693252db4962c0b61b445c626faa0693827b1b Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 14:31:31 -0700 Subject: [PATCH 14/34] fix(gpu): mark AMD driver-install RUN as --privileged amdgpu-dkms's ./configure runs a Kbuild probe to detect CFLAGS. In buildkit's default unprivileged sandbox that probe fails with configure: error: cannot detect CFLAGS... grep: amd/dkms/config/config.h: No such file or directory make: *** No rule to make target 'amd/dkms/config/config.h' Reproduced against a plain `docker run --privileged` baseline: same script, same base image (kairos-ubuntu:24.04-core-amd64-generic-v4.0.4), same kernel (6.17.0-35-generic), same AMDGPU_DRIVER_RELEASE=7.2.1 -- succeeds end-to-end with modules landing under /lib/modules//updates/dkms/. The only observed difference is --privileged, so mark the AMD RUN step explicitly. The wrapper (./earthly.sh) already passes --allow-privileged, and other RUN steps in this repo use --privileged similarly, so no downstream change needed. NVIDIA's driver install doesn't use the same ./configure sanity-check step and continues to work unprivileged; its RUN block is left as-is. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 32903267..46c1df04 100644 --- a/Earthfile +++ b/Earthfile @@ -853,7 +853,17 @@ base-image: IF [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh - RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ + # --privileged is required for AMD's amdgpu-dkms ./configure step: + # its "detect CFLAGS" probe invokes Kbuild in a way that fails in + # buildkit's default unprivileged sandbox with + # "configure: error: cannot detect CFLAGS..." + # "make: *** No rule to make target 'amd/dkms/config/config.h'" + # Reproduced against the plain `docker run --privileged` baseline + # succeeding on the same host with the same script + base image. + # Caller must pass --allow-privileged to earthly (already required + # by other steps in this repo). + RUN --privileged \ + chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ From ead23e8a9a10ee5602656c26896d989af65153b1 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 14:39:30 -0700 Subject: [PATCH 15/34] fix(gpu): add full Kbuild bootstrap deps before amdgpu-dkms install amdgpu-dkms's ./configure fails silently at "cannot detect CFLAGS..." when Kbuild can't complete the `make -f -` heredoc probe it uses to extract KBUILD_CFLAGS. build-essential covers gcc/make/libc but modern kernel Makefiles unconditionally reach for bc, bison, flex, libelf-dev, libssl-dev, and pahole (from dwarves). --no-install-recommends skips all of those. Enumerate them explicitly to keep the image lean without breaking the AMD DKMS build. dwarves is AMD's own `Recommends:` on amdgpu-dkms -- it just wasn't pulled because we use --no-install-recommends. cpio + xz-utils are added defensively for the initramfs trigger that runs after DKMS install. Co-Authored-By: Claude Opus 4.7 --- scripts/install-amdgpu-drivers.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index d89c3b9f..f943e0f9 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -156,10 +156,23 @@ fi # --------------------------------------------------------------------------- log "Installing build toolchain ..." apt-get update || true +# The full Kbuild bootstrap: gcc/make/libc from build-essential PLUS the +# tools that recent kernel Makefiles pull in unconditionally. Missing any of +# these fails AMD's amdgpu-dkms ./configure at "cannot detect CFLAGS..." -- +# the failure mode is silent because CFLAGS-detection just runs `make -f -` +# and swallows stderr. We enumerate them explicitly instead of relying on +# --install-recommends (which would also pull other unwanted docs/data). +# bc, bison, flex : referenced by kernel Kbuild machinery +# libelf-dev : module utilities (modpost) + BPF +# libssl-dev : signing certificates / hash routines +# dwarves : pahole for BTF debuginfo (amdgpu-dkms explicitly Recommends this) +# cpio, xz-utils : initramfs assembly (may be needed by initramfs-tools trigger) apt-get install -y --no-install-recommends \ ca-certificates curl wget gnupg \ build-essential gcc make \ dkms kmod libc6-dev initramfs-tools \ + bc bison flex libelf-dev libssl-dev dwarves \ + cpio xz-utils \ || die "failed to install build toolchain." # --------------------------------------------------------------------------- From 5b63b3819d0ec61ccc4126b1857df51e811d8abe Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:07:21 -0700 Subject: [PATCH 16/34] feat(gpu): add scripts/prebuild-amdgpu-artifact.sh (host-side DKMS build) Compiles amdgpu-dkms against the kairos base image's kernel in a host-side `docker run --privileged` container, then tars the produced kernel module + firmware + config drop-ins into build/amdgpu-artifact---.tar.gz. The main Earthly build consumes that artifact via COPY, sidestepping the buildkit RUN sandbox limitation that breaks AMD's ./configure heredoc probe. Cache by (base-image digest x kernel version x AMDGPU_DRIVER_RELEASE); repeat builds skip the ~8-10 min compile. Cache miss reproduces the same behavior we already verified end-to-end via direct docker run on the jump host. Prints "AMDGPU_ARTIFACT_PATH=" on the last stdout line so the earthly.sh wrapper (next commit) can pick it up and thread it into Earthly. Standalone; no other files touched by this commit. Co-Authored-By: Claude Opus 4.7 --- scripts/prebuild-amdgpu-artifact.sh | 151 ++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 scripts/prebuild-amdgpu-artifact.sh diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh new file mode 100755 index 00000000..a2d64232 --- /dev/null +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# prebuild-amdgpu-artifact.sh +# +# Compile AMD's amdgpu-dkms module against the kairos base image's kernel in a +# host-side `docker run --privileged` container, then tar the resulting kernel +# module + firmware + support files into an artifact the main Earthly build +# COPYs in. +# +# WHY THIS EXISTS +# --------------- +# Earthly's buildkit RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc +# probe (fails at "cannot detect CFLAGS..."), despite the same script + same +# base image + same host succeeding under plain `docker run --privileged`. +# The specific buildkit-vs-docker sandbox difference is not something we +# control from the Earthfile. Instead of fighting it, we run the DKMS build +# outside Earthly, in the environment we know works, and let Earthly consume +# the produced artifact via COPY. +# +# WHAT ENDS UP IN THE ARTIFACT +# /lib/modules//updates/dkms/.ko* +# /lib/firmware/amdgpu/* (firmware blobs) +# /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf (defensive) +# /etc/modules-load.d/amdgpu.conf (autoload) +# /etc/canvos/amdgpu-driver-source (on-node marker) +# +# Extracting the tar into the image (via the Earthfile) + running depmod on +# the target kernel is functionally equivalent to running install-amdgpu- +# drivers.sh directly in the image. +# +# CACHING +# Artifacts are stored at build/amdgpu-artifact---.tar.gz +# Cache hits when release + base-image digest + kver match. Docker image +# pulls are hit via the local docker daemon's own cache. +# +# INPUTS (env vars; defaults mirror the Earthfile / .arg.template) +# BASE_IMAGE kairos base image ref (REQUIRED) +# AMDGPU_DRIVER_RELEASE default: 7.2.1 (pairs with GPU Operator v1.5.0) +# AMDGPU_ARTIFACT_DIR default: ./build +# AMDGPU_FORCE_REBUILD set to 1 to bypass cache +# +# OUTPUT (stdout) +# Absolute path to the produced .tar.gz on the last line, prefixed by +# "AMDGPU_ARTIFACT_PATH=" so callers can `eval "$(prebuild-amdgpu-artifact.sh)"` +# or just take the last line. +# +set -euo pipefail + +log() { echo "[prebuild-amdgpu] $*" >&2; } +die() { echo "[prebuild-amdgpu] ERROR: $*" >&2; exit 1; } + +# --- inputs --------------------------------------------------------------- +: "${BASE_IMAGE:?BASE_IMAGE must be set (kairos base image ref, e.g. us-docker.pkg.dev/palette-images/edge/kairos-ubuntu:24.04-core-amd64-generic-v4.0.4)}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" +AMDGPU_ARTIFACT_DIR="${AMDGPU_ARTIFACT_DIR:-./build}" +AMDGPU_FORCE_REBUILD="${AMDGPU_FORCE_REBUILD:-0}" + +command -v docker >/dev/null 2>&1 || die "docker must be available on the build host." + +# The install script we'll run inside the container. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +INSTALL_SCRIPT="${SCRIPT_DIR}/install-amdgpu-drivers.sh" +HEADERS_SCRIPT="${SCRIPT_DIR}/install-kernel-headers.sh" +[ -r "${INSTALL_SCRIPT}" ] || die "cannot find install-amdgpu-drivers.sh at ${INSTALL_SCRIPT}" +[ -r "${HEADERS_SCRIPT}" ] || die "cannot find install-kernel-headers.sh at ${HEADERS_SCRIPT}" + +mkdir -p "${AMDGPU_ARTIFACT_DIR}" + +# --- discover target kernel + base image digest --------------------------- +log "Pulling base image (may be cached): ${BASE_IMAGE}" +docker pull "${BASE_IMAGE}" >/dev/null || die "failed to pull ${BASE_IMAGE}" + +BASE_DIGEST="$(docker image inspect -f '{{.Id}}' "${BASE_IMAGE}" | sed 's/^sha256://' | cut -c1-12)" +[ -n "${BASE_DIGEST}" ] || die "could not read digest of ${BASE_IMAGE}" + +KVER="$(docker run --rm --entrypoint /bin/sh "${BASE_IMAGE}" -c 'ls /lib/modules | sort -V | tail -1' 2>/dev/null)" +[ -n "${KVER}" ] || die "could not discover kernel from /lib/modules inside ${BASE_IMAGE}" + +ARTIFACT_NAME="amdgpu-artifact-${AMDGPU_DRIVER_RELEASE}-${KVER}-${BASE_DIGEST}.tar.gz" +ARTIFACT_PATH="$(cd "${AMDGPU_ARTIFACT_DIR}" && pwd)/${ARTIFACT_NAME}" + +log "Base image digest : ${BASE_DIGEST}" +log "Target kernel : ${KVER}" +log "Driver release : ${AMDGPU_DRIVER_RELEASE}" +log "Artifact path : ${ARTIFACT_PATH}" + +# --- cache check ---------------------------------------------------------- +if [ "${AMDGPU_FORCE_REBUILD}" != "1" ] && [ -s "${ARTIFACT_PATH}" ]; then + log "Cache hit -- reusing existing artifact. Set AMDGPU_FORCE_REBUILD=1 to override." + echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" + exit 0 +fi + +# --- build ---------------------------------------------------------------- +# Run the install script inside a privileged container against the same base +# image the Earthfile will use, then tar out the produced files. We stream +# the tar over stdout to avoid needing an intermediate volume mount that some +# rootless docker setups can't do cleanly. +STAGE_DIR="$(mktemp -d "${AMDGPU_ARTIFACT_DIR}/.amdgpu-build.XXXXXX")" +trap 'rm -rf "${STAGE_DIR}"' EXIT + +log "Compiling amdgpu-dkms in container. This takes ~8-10 min the first time." +docker run --rm --privileged \ + -e AMDGPU_DRIVER_SOURCE=dkms \ + -e AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE}" \ + -e AMDGPU_REBUILD_INITRD=false \ + -e KVER_EXPECTED="${KVER}" \ + -v "${INSTALL_SCRIPT}:/tmp/install-amdgpu-drivers.sh:ro" \ + -v "${HEADERS_SCRIPT}:/tmp/install-kernel-headers.sh:ro" \ + --entrypoint /bin/bash \ + "${BASE_IMAGE}" \ + -c ' + set -eo pipefail + chmod +x /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + /tmp/install-amdgpu-drivers.sh 1>&2 + + # Verify the module actually landed. + MODDIR="/lib/modules/${KVER_EXPECTED}/updates/dkms" + if ! find "${MODDIR}" -name "amdgpu.ko*" 2>/dev/null | grep -q .; then + echo "prebuild: no amdgpu module under ${MODDIR}" >&2 + exit 1 + fi + + # Build the tar to stdout. Paths must exist to be included; the tar + # is anchored at / so extraction inside the image lands under the + # same absolute paths. + TAR_INPUTS=( + "/lib/modules/${KVER_EXPECTED}/updates/dkms" + "/etc/modules-load.d/amdgpu.conf" + "/etc/canvos/amdgpu-driver-source" + ) + # Firmware + framework drop-in are optional but helpful; skip silently if absent. + [ -d /lib/firmware/amdgpu ] && TAR_INPUTS+=("/lib/firmware/amdgpu") + [ -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf ] && \ + TAR_INPUTS+=("/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf") + + tar -czf - "${TAR_INPUTS[@]}" + ' > "${STAGE_DIR}/artifact.tar.gz" + +# Sanity: tar file must be non-empty and contain the amdgpu module. +[ -s "${STAGE_DIR}/artifact.tar.gz" ] || die "prebuild produced an empty artifact." +if ! tar -tzf "${STAGE_DIR}/artifact.tar.gz" | grep -q "updates/dkms/amdgpu.ko"; then + die "artifact does not contain an amdgpu module under updates/dkms/; \ +inspect ${STAGE_DIR}/artifact.tar.gz." +fi + +mv "${STAGE_DIR}/artifact.tar.gz" "${ARTIFACT_PATH}" +log "Artifact produced ($(du -h "${ARTIFACT_PATH}" | awk '{print $1}'))." + +# --- output --------------------------------------------------------------- +echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" From 6b4d8babd4f13181f490d29011354a9a3973aa31 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:09:32 -0700 Subject: [PATCH 17/34] feat(gpu): hook AMD driver prebuild into earthly.sh wrapper Before dispatching to earthly (build_with_proxy/build_without_proxy), detect INSTALL_AMD_GPU_DRIVERS=true + AMDGPU_DRIVER_SOURCE=dkms from either .arg or CLI overrides, derive BASE_IMAGE via the same tag formula the Earthfile uses, and invoke scripts/prebuild-amdgpu-artifact.sh to compile the amdgpu module on the host. The last line of prebuild output is parsed for the artifact path; it's threaded to earthly as --AMDGPU_ARTIFACT_PATH=, which the companion Earthfile change consumes via COPY + tar-extract. Inbox mode skips this step entirely. Non-AMD builds skip this step entirely. Existing CLI/.arg semantics for INSTALL_AMD_GPU_DRIVERS, AMDGPU_DRIVER_SOURCE, AMDGPU_DRIVER_RELEASE, and BASE_IMAGE are preserved unchanged. Ubuntu-only guard: prebuild refuses OS_DISTRIBUTION != ubuntu with a clear error, mirroring the mutual-exclusion / Ubuntu-only checks already in the Earthfile for the AMD block. Co-Authored-By: Claude Opus 4.7 --- earthly.sh | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/earthly.sh b/earthly.sh index 9c50e22d..68fabdfa 100755 --- a/earthly.sh +++ b/earthly.sh @@ -309,6 +309,86 @@ if [[ "$1" == "+maas-image" ]]; then exit 0 fi +# --------------------------------------------------------------------------- +# AMD GPU driver prebuild (dkms mode only). +# +# amdgpu-dkms's ./configure fails inside Earthly's buildkit RUN sandbox +# (see docs/amd-gpu-airgapped.md). We pre-compile the module in a plain +# `docker run --privileged` container here on the build host -- the exact +# environment we verified works end-to-end -- and pass the resulting tarball +# to Earthly for a simple COPY + tar-extract + depmod inside the base image. +# --------------------------------------------------------------------------- + +# Read a --FOO=bar override out of $@ without consuming it. Prints the value +# or empty; the arg is still passed through to earthly untouched. +peek_arg() { + local key="$1" + local a + for a in "$@"; do + case "$a" in + --${key}=*) printf '%s' "${a#--${key}=}"; return ;; + esac + done +} + +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="$(peek_arg INSTALL_AMD_GPU_DRIVERS "$@")" +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="${INSTALL_AMD_GPU_DRIVERS_EFFECTIVE:-${INSTALL_AMD_GPU_DRIVERS:-false}}" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_SOURCE "$@")" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="${AMDGPU_DRIVER_SOURCE_EFFECTIVE:-${AMDGPU_DRIVER_SOURCE:-dkms}}" + +if [ "$INSTALL_AMD_GPU_DRIVERS_EFFECTIVE" = "true" ] && [ "$AMDGPU_DRIVER_SOURCE_EFFECTIVE" = "dkms" ]; then + # Derive the same BASE_IMAGE the Earthfile derives, so we compile against + # the same rootfs Earthly is about to build on. Only Ubuntu is supported + # for AMD driver pre-install; the AMD mutual-exclusion + Ubuntu-only checks + # elsewhere handle other OS_DISTRIBUTIONs. + AMDGPU_BASE_IMAGE="$(peek_arg BASE_IMAGE "$@")" + AMDGPU_BASE_IMAGE="${AMDGPU_BASE_IMAGE:-${BASE_IMAGE:-}}" + if [ -z "$AMDGPU_BASE_IMAGE" ]; then + _os_dist="$(peek_arg OS_DISTRIBUTION "$@")"; _os_dist="${_os_dist:-${OS_DISTRIBUTION:-ubuntu}}" + _os_ver="$(peek_arg OS_VERSION "$@")"; _os_ver="${_os_ver:-${OS_VERSION:-24.04}}" + _arch="$(peek_arg ARCH "$@")"; _arch="${_arch:-${ARCH:-amd64}}" + _kairos_ver="$(peek_arg KAIROS_VERSION "$@")"; _kairos_ver="${_kairos_ver:-${KAIROS_VERSION:-v4.0.4}}" + _kairos_url="$(peek_arg KAIROS_BASE_IMAGE_URL "$@")"; _kairos_url="${_kairos_url:-${KAIROS_BASE_IMAGE_URL:-$SPECTRO_PUB_REPO/edge}}" + _is_uki="$(peek_arg IS_UKI "$@")"; _is_uki="${_is_uki:-${IS_UKI:-false}}" + + if [ "$_os_dist" != "ubuntu" ]; then + echo "AMD GPU driver pre-install requires OS_DISTRIBUTION=ubuntu (got: $_os_dist)." >&2 + exit 1 + fi + # Same tag formula as Earthfile lines ~141-151. + if [ "$_os_ver" = "22" ] || [ "$_os_ver" = "20" ]; then + _tag="kairos-${_os_dist}:${_os_ver}.04-core-${_arch}-generic-${_kairos_ver}" + elif [ "$_is_uki" = "true" ]; then + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}-uki" + else + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}" + fi + AMDGPU_BASE_IMAGE="${_kairos_url}/${_tag}" + fi + + AMDGPU_DRIVER_RELEASE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_RELEASE "$@")" + AMDGPU_DRIVER_RELEASE_EFFECTIVE="${AMDGPU_DRIVER_RELEASE_EFFECTIVE:-${AMDGPU_DRIVER_RELEASE:-7.2.1}}" + + echo "=== Pre-building AMD amdgpu driver (dkms mode) ===" + echo " BASE_IMAGE: $AMDGPU_BASE_IMAGE" + echo " AMDGPU_DRIVER_RELEASE: $AMDGPU_DRIVER_RELEASE_EFFECTIVE" + prebuild_out="$( + BASE_IMAGE="$AMDGPU_BASE_IMAGE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE_EFFECTIVE" \ + AMDGPU_ARTIFACT_DIR="$(pwd)/build" \ + bash scripts/prebuild-amdgpu-artifact.sh + )" || { echo "AMD driver pre-build failed. See lines above." >&2; exit 1; } + + # Last line of prebuild output is: AMDGPU_ARTIFACT_PATH= + AMDGPU_ARTIFACT_PATH="$(printf '%s\n' "$prebuild_out" | tail -1 | sed -n 's/^AMDGPU_ARTIFACT_PATH=//p')" + [ -s "$AMDGPU_ARTIFACT_PATH" ] || { echo "Prebuild did not emit AMDGPU_ARTIFACT_PATH; aborting." >&2; exit 1; } + echo " Artifact: $AMDGPU_ARTIFACT_PATH" + + # Thread the artifact path through to Earthly. Its Earthfile ARG (added in + # the companion commit) picks this up and consumes the tarball. + set -- "$@" "--AMDGPU_ARTIFACT_PATH=$AMDGPU_ARTIFACT_PATH" +fi + # Normal build flow for other targets if [ -z "$HTTP_PROXY" ] && [ -z "$HTTPS_PROXY" ] && [ -z "$(find certs -type f ! -name '.*' -print -quit)" ]; then build_without_proxy "$@" From 8d90dc22891f1e02d2c6b999f7d6cbb21a67f2af Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:11:37 -0700 Subject: [PATCH 18/34] feat(gpu): consume AMD driver artifact in Earthfile + install script + docs Complete the prebuild-artifact flow started in the previous two commits. Earthfile: * new ARG AMDGPU_ARTIFACT_PATH (empty default) * base-image AMD block now branches on (AMDGPU_DRIVER_SOURCE, artifact): - dkms + artifact present -> COPY tarball, extract, depmod, initrd (fast path used by ./earthly.sh + prebuild helper -- the release path) - inbox, or dkms without artifact -> unchanged legacy path (works under `docker run --privileged`; fails under buildkit's RUN sandbox at AMD's ./configure -- documented) scripts/install-amdgpu-drivers.sh: * new AMDGPU_ARTIFACT_PATH fast path before the in-buildkit DKMS block: extract tarball, verify updates/dkms/amdgpu.ko*, depmod, rebuild initrd, write the on-node marker at /etc/canvos/amdgpu-driver-source. In-buildkit DKMS legacy path retained for direct `docker run` invocations and for scripts/prebuild-amdgpu-artifact.sh's own inner container. * docstring updated to describe the three execution paths. docs/amd-gpu-airgapped.md: * new "two-stage build" explanation covering why the prebuild exists (buildkit RUN sandbox breaks AMD ./configure) and how it's wired up behind ./earthly.sh. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 57 ++++++++++++++------- docs/amd-gpu-airgapped.md | 51 +++++++++++-------- scripts/install-amdgpu-drivers.sh | 85 +++++++++++++++++++++++++++---- 3 files changed, 144 insertions(+), 49 deletions(-) diff --git a/Earthfile b/Earthfile index 46c1df04..a41f3a7a 100644 --- a/Earthfile +++ b/Earthfile @@ -89,6 +89,12 @@ ARG AMDGPU_DRIVER_SOURCE=dkms # marker (30.30.1, 31.30) URL segments; either form is accepted here. The 31.x line is # tech-preview -- do not mix with a production operator. See docs/amd-gpu-airgapped.md. ARG AMDGPU_DRIVER_RELEASE=7.2.1 +# Path to a driver artifact produced by scripts/prebuild-amdgpu-artifact.sh +# on the build host. Threaded in by earthly.sh when INSTALL_AMD_GPU_DRIVERS=true +# and AMDGPU_DRIVER_SOURCE=dkms. When set, the base-image AMD block skips the +# in-buildkit DKMS install (which fails in buildkit's RUN sandbox -- see docs) +# and simply extracts the pre-built modules + firmware + config drop-ins. +ARG AMDGPU_ARTIFACT_PATH="" ARG AMDGPU_REBUILD_INITRD=true # NVIDIA and AMD driver pre-install are mutually exclusive within a single image. @@ -851,24 +857,39 @@ base-image: # AMD Instinct GPU driver (amdgpu-dkms) + kernel module, built against the # now-finalized image kernel. Mutually exclusive with the NVIDIA block above. IF [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] - COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh - COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh - # --privileged is required for AMD's amdgpu-dkms ./configure step: - # its "detect CFLAGS" probe invokes Kbuild in a way that fails in - # buildkit's default unprivileged sandbox with - # "configure: error: cannot detect CFLAGS..." - # "make: *** No rule to make target 'amd/dkms/config/config.h'" - # Reproduced against the plain `docker run --privileged` baseline - # succeeding on the same host with the same script + base image. - # Caller must pass --allow-privileged to earthly (already required - # by other steps in this repo). - RUN --privileged \ - chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ - AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ - AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ - AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ - /tmp/install-amdgpu-drivers.sh && \ - rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + # dkms mode with a pre-built artifact (default path when earthly.sh + # produced one via scripts/prebuild-amdgpu-artifact.sh). Buildkit's + # RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe -- + # see docs/amd-gpu-airgapped.md. The prebuild runs on the host in a + # plain `docker run --privileged` against the same base image, and + # this stage just extracts the resulting modules + firmware + drop-ins. + IF [ "$AMDGPU_DRIVER_SOURCE" = "dkms" ] && [ "$AMDGPU_ARTIFACT_PATH" != "" ] + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + COPY "$AMDGPU_ARTIFACT_PATH" /tmp/amdgpu-artifact.tar.gz + RUN --privileged \ + chmod 755 /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE=dkms \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_ARTIFACT_PATH=/tmp/amdgpu-artifact.tar.gz \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/amdgpu-artifact.tar.gz + ELSE + # inbox mode, OR dkms mode without a pre-built artifact (which + # will fail in buildkit's sandbox, but we let install-amdgpu- + # drivers.sh emit its own clear error rather than short-circuit + # here). install-kernel-headers.sh is only needed for the + # in-buildkit DKMS path; inbox mode doesn't use it. + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + RUN --privileged \ + chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + END END IF [ "$CIS_HARDENING" = "true" ] diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md index ab2ef767..4a3c48ae 100644 --- a/docs/amd-gpu-airgapped.md +++ b/docs/amd-gpu-airgapped.md @@ -68,31 +68,42 @@ needed at boot**. ## The key build-time problem this solves (dkms mode) Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, not -the kernel baked into the image. In `dkms` mode the script therefore: - -1. derives the **target kernel** from `/lib/modules/*`, -2. installs **ABI-exact kernel headers** for it (reusing - [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh)) plus - `linux-modules-extra-`, -3. registers the AMD driver apt repo via the release-matched `amdgpu-install` - package (auto-discovered from `repo.radeon.com/amdgpu-install//`), -4. installs `amdgpu-dkms` and forces **DKMS build + install + `depmod`** against - the target kernel, and -5. **verifies** an `amdgpu.ko` landed under - `/lib/modules//updates/dkms/` **and** that `dkms status` - reports the module as `installed` for that kernel — the in-tree module - shipped under `kernel/…` is *not* accepted. Failing either check aborts - the image build with a pointer at `AMDGPU_DRIVER_RELEASE` and the inbox - fallback. - -It runs in the `base-image` target **after** the kernel is finalized. +the kernel baked into the image. In `dkms` mode the driver must be compiled +against the image kernel's headers. + +We ran into a second problem too: Earthly's buildkit `RUN` sandbox breaks +AMD's `amdgpu-dkms` `./configure` heredoc probe with +`"cannot detect CFLAGS…"`, even though the same script + base image + host +succeed under plain `docker run --privileged`. Rather than debug buildkit +(some seccomp/apparmor/mount detail we don't control from the Earthfile), +`dkms` mode uses a **two-stage build**: + +1. **Prebuild on the host** (via `scripts/prebuild-amdgpu-artifact.sh`, + auto-invoked by `./earthly.sh`). Runs a plain `docker run --privileged` + against the same kairos base image, executes the DKMS install inside, + tars the resulting `/lib/modules//updates/dkms/` + firmware + + drop-ins into `build/amdgpu-artifact---.tar.gz`. + Cached by (release × base-image digest × kver); ~8–10 min the first + time, instant on cache hit. +2. **Consume in Earthly**: the `base-image` target `COPY`s the tarball and + extracts it, runs `depmod` against the image kernel, and rebuilds the + initrd. No `./configure`, no compile inside buildkit. + +Both stages use the same `scripts/install-amdgpu-drivers.sh` — the in-buildkit +step just takes the `AMDGPU_ARTIFACT_PATH` fast path. The result on-node is +identical to a native DKMS install. + +`inbox` mode skips both stages entirely and only ensures the in-tree amdgpu +autoloads. > **No blacklist needed.** Unlike NVIDIA (where `nouveau` must be blacklisted), > the DKMS `amdgpu` module replaces the in-tree one via `depmod`'s `updates/` > override. The script just autoloads `amdgpu`. -In `inbox` mode steps 2–5 are skipped entirely; the script only ensures -`amdgpu` autoloads and rebuilds the initrd. +> **When is the prebuild helper invoked?** `./earthly.sh` triggers it +> automatically when `INSTALL_AMD_GPU_DRIVERS=true` and +> `AMDGPU_DRIVER_SOURCE=dkms` (either from `.arg` or a CLI override). No +> extra command needed. It's skipped for `inbox`, or for any non-AMD build. --- diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index f943e0f9..7fd364dc 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -6,20 +6,24 @@ # Ubuntu base image, so a node booted from the image can run the AMD GPU Operator # with `driver.enable=false` in a fully air-gapped environment. # -# TWO MODES (AMDGPU_DRIVER_SOURCE) -# -------------------------------- -# dkms (default) -- download AMD's amdgpu-dkms source from repo.radeon.com and -# DKMS-build it against the image kernel. Recommended for -# Instinct/MI silicon, where the in-tree driver typically -# lags on SMU firmware interfaces and per-SKU features. +# MODES (AMDGPU_DRIVER_SOURCE) +# ---------------------------- +# dkms (default) -- install AMD's amdgpu-dkms. Two execution paths: +# (a) if AMDGPU_ARTIFACT_PATH is set (Earthly build via +# earthly.sh's prebuild helper), extract the pre- +# compiled modules + firmware + drop-ins from the +# tarball, depmod, rebuild initrd. Fast. +# (b) otherwise download from repo.radeon.com and +# DKMS-build against the image kernel in-place. +# Works under `docker run --privileged`; FAILS in +# Earthly's buildkit RUN sandbox at AMD's ./configure +# step -- use path (a) for Earthly. # inbox -- do NOT install the AMD apt repo or amdgpu-dkms. Rely on # the in-tree `amdgpu` module that Ubuntu ships with # linux-modules-$(uname -r) and the firmware blobs in # linux-firmware. Only ensures amdgpu autoloads. Choose -# this when the DKMS build fails against your image kernel -# (e.g. AMD hasn't published a driver release supporting -# your kernel yet) and you accept the in-tree driver's -# feature set (may miss recent SMU / per-SKU support). +# this when you accept the in-tree driver's feature set +# (may miss recent SMU / per-SKU support). # # WHAT THIS COVERS (dkms mode, OS side only) # ------------------------------------------ @@ -148,7 +152,66 @@ EOF fi # --------------------------------------------------------------------------- -# DKMS MODE begins here. +# DKMS MODE with a pre-built artifact (produced by +# scripts/prebuild-amdgpu-artifact.sh on the build host). +# +# Buildkit's RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe, +# so we compile outside Earthly in `docker run --privileged` and consume the +# resulting tarball here. Structurally this branch just extracts the tarball, +# runs depmod against the target kernel, and rebuilds the initrd. +# --------------------------------------------------------------------------- +AMDGPU_ARTIFACT_PATH="${AMDGPU_ARTIFACT_PATH:-}" +if [ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && [ -n "${AMDGPU_ARTIFACT_PATH}" ]; then + log "dkms mode: consuming pre-built artifact ${AMDGPU_ARTIFACT_PATH}" + [ -s "${AMDGPU_ARTIFACT_PATH}" ] || die "AMDGPU_ARTIFACT_PATH='${AMDGPU_ARTIFACT_PATH}' \ +is not a non-empty file inside the build container. Verify the earthly.sh \ +wrapper produced it and Earthly COPYed it in." + + log "Extracting artifact into root filesystem ..." + tar -xzf "${AMDGPU_ARTIFACT_PATH}" -C / \ + || die "tar extraction of ${AMDGPU_ARTIFACT_PATH} failed." + + MODDIR="/lib/modules/${KVER}" + if ! find "${MODDIR}/updates/dkms" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' >&2 || true + die "amdgpu module missing under ${MODDIR}/updates/dkms after extract. \ +Artifact was built for a different kernel? Delete build/amdgpu-artifact-*.tar.gz \ +and rebuild (AMDGPU_FORCE_REBUILD=1) or verify BASE_IMAGE matches." + fi + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" || die "depmod failed for ${KVER}." + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + # Marker written by the prebuild is preserved from the tar. Overwrite + # any prebuild-mode marker with the final in-image reality. + printf 'AMDGPU_DRIVER_SOURCE=dkms (artifact)\nAMDGPU_DRIVER_RELEASE=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}, artifact) baked in for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE in-buildkit (fallback). Runs the full apt + DKMS build inside +# the image. This path fails inside Earthly's buildkit RUN sandbox at AMD's +# ./configure step (see docs) but is retained for: +# - direct `docker run --privileged` invocations (proven working), +# - the scripts/prebuild-amdgpu-artifact.sh helper, which uses this same +# script inside the container it spawns. # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- From 583713d80c089da677561cbfe0fac60ad21db259 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:13:50 -0700 Subject: [PATCH 19/34] fix(gpu): invoke install script via bash (RO bind mount can't be chmod'd) The prebuild's inner container bind-mounts the install scripts read-only (-v ...:ro). chmod +x fails on RO mounts. Bash can execute scripts without +x when invoked as `bash `; no functional difference otherwise. Co-Authored-By: Claude Opus 4.7 --- scripts/prebuild-amdgpu-artifact.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh index a2d64232..5b03145c 100755 --- a/scripts/prebuild-amdgpu-artifact.sh +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -111,8 +111,9 @@ docker run --rm --privileged \ "${BASE_IMAGE}" \ -c ' set -eo pipefail - chmod +x /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh - /tmp/install-amdgpu-drivers.sh 1>&2 + # Scripts are bind-mounted read-only from the host; invoke via `bash` + # so we don t need chmod +x (which would fail on the RO mount). + bash /tmp/install-amdgpu-drivers.sh 1>&2 # Verify the module actually landed. MODDIR="/lib/modules/${KVER_EXPECTED}/updates/dkms" From 4a163f28187d306f05253004e9e6abb27af0c939 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:28:32 -0700 Subject: [PATCH 20/34] fix(gpu): preserve artifact on verify failure + dump tar listing Previous verify path deleted the tarball via STAGE_DIR trap before the user could inspect it. Now: * mv to final path FIRST (before verify), so any failure leaves the file behind for inspection * on verify failure, dump the first 40 tar entries to stderr so we can see what path structure actually landed in the archive * grep pattern loosened to `updates/dkms/.*amdgpu\.ko` (regex) in case tar mangled paths in an unexpected way Co-Authored-By: Claude Opus 4.7 --- scripts/prebuild-amdgpu-artifact.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh index 5b03145c..e341d173 100755 --- a/scripts/prebuild-amdgpu-artifact.sh +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -138,14 +138,22 @@ docker run --rm --privileged \ tar -czf - "${TAR_INPUTS[@]}" ' > "${STAGE_DIR}/artifact.tar.gz" -# Sanity: tar file must be non-empty and contain the amdgpu module. +# Move the artifact to its final path IMMEDIATELY so a subsequent verify +# failure leaves a diagnosable file behind (STAGE_DIR is wiped by the trap). [ -s "${STAGE_DIR}/artifact.tar.gz" ] || die "prebuild produced an empty artifact." -if ! tar -tzf "${STAGE_DIR}/artifact.tar.gz" | grep -q "updates/dkms/amdgpu.ko"; then - die "artifact does not contain an amdgpu module under updates/dkms/; \ -inspect ${STAGE_DIR}/artifact.tar.gz." +mv "${STAGE_DIR}/artifact.tar.gz" "${ARTIFACT_PATH}" + +# Sanity: tar file must be a valid gzip and contain the amdgpu module. +tar_listing="$(tar -tzf "${ARTIFACT_PATH}" 2>&1)" \ + || die "artifact ${ARTIFACT_PATH} is not a valid gzipped tar. Head of output: $(printf '%s\n' "${tar_listing}" | head -3)" +if ! printf '%s\n' "${tar_listing}" | grep -qE "updates/dkms/.*amdgpu\.ko"; then + log "Artifact contents (first 40 entries):" + printf '%s\n' "${tar_listing}" | head -40 | sed 's/^/ /' >&2 + die "artifact ${ARTIFACT_PATH} does not contain an amdgpu module under updates/dkms/. \ +The tarball is preserved for inspection. Delete it and rerun with \ +AMDGPU_FORCE_REBUILD=1 to try again." fi -mv "${STAGE_DIR}/artifact.tar.gz" "${ARTIFACT_PATH}" log "Artifact produced ($(du -h "${ARTIFACT_PATH}" | awk '{print $1}'))." # --- output --------------------------------------------------------------- From c22370a1198de16711d8bc096df1b6a7085482f9 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:48:25 -0700 Subject: [PATCH 21/34] fix(gpu): pass artifact path repo-relative to Earthly (COPY reads build context) Earthly's COPY expects paths relative to the Earthfile's directory (the repo root), not absolute host paths. The prebuild helper emits an absolute path for its own file-existence checks; the wrapper now converts that to a repo-relative path before threading it into Earthly as --AMDGPU_ARTIFACT_PATH=build/amdgpu-artifact-...tar.gz. Fails fast with a clear message if the artifact somehow lands outside the repo tree. Co-Authored-By: Claude Opus 4.7 --- earthly.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/earthly.sh b/earthly.sh index 68fabdfa..d18ff737 100755 --- a/earthly.sh +++ b/earthly.sh @@ -384,9 +384,19 @@ if [ "$INSTALL_AMD_GPU_DRIVERS_EFFECTIVE" = "true" ] && [ "$AMDGPU_DRIVER_SOURCE [ -s "$AMDGPU_ARTIFACT_PATH" ] || { echo "Prebuild did not emit AMDGPU_ARTIFACT_PATH; aborting." >&2; exit 1; } echo " Artifact: $AMDGPU_ARTIFACT_PATH" - # Thread the artifact path through to Earthly. Its Earthfile ARG (added in - # the companion commit) picks this up and consumes the tarball. - set -- "$@" "--AMDGPU_ARTIFACT_PATH=$AMDGPU_ARTIFACT_PATH" + # Earthly's COPY reads from the repo build-context (the directory containing + # the Earthfile), not from the host filesystem, so we must pass a path + # relative to the repo root -- not the absolute host path. + repo_root="$(pwd)" + case "$AMDGPU_ARTIFACT_PATH" in + "$repo_root"/*) AMDGPU_ARTIFACT_REL="${AMDGPU_ARTIFACT_PATH#$repo_root/}" ;; + *) echo "Prebuild artifact '$AMDGPU_ARTIFACT_PATH' is outside repo root '$repo_root'; \ +COPY into Earthly would fail. Move the artifact under the repo tree." >&2 ; exit 1 ;; + esac + + # Thread the (repo-relative) artifact path through to Earthly. Its Earthfile + # ARG (added in the companion commit) picks this up and consumes the tarball. + set -- "$@" "--AMDGPU_ARTIFACT_PATH=$AMDGPU_ARTIFACT_REL" fi # Normal build flow for other targets From a21a6cb7275f95b632c51f48ed79e55fd6ef11aa Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 11 Jul 2026 15:57:53 -0700 Subject: [PATCH 22/34] fix(gpu): whitelist AMD driver artifact in .earthlyignore `build/*` in .earthlyignore was hiding the pre-built AMD driver tarball from Earthly's build context, so COPY build/amdgpu-artifact-...tar.gz failed with "no such file or directory" even though the artifact existed on the host. Add a negation pattern for build/amdgpu-artifact-*.tar.gz so this specific artifact is visible while everything else under build/ (Earthly SAVE ARTIFACT outputs, etc.) stays ignored. Co-Authored-By: Claude Opus 4.7 --- .earthlyignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.earthlyignore b/.earthlyignore index c6fb21c9..486f781e 100644 --- a/.earthlyignore +++ b/.earthlyignore @@ -1,2 +1,6 @@ local/ -build/* \ No newline at end of file +build/* +# Whitelist AMD driver artifacts (produced by scripts/prebuild-amdgpu-artifact.sh +# and consumed by the base-image target via COPY). Without this exception, +# `build/*` above would hide the tarball from Earthly's build context. +!build/amdgpu-artifact-*.tar.gz \ No newline at end of file From e54b3a53d008cbf6aa82a1f6f2c728d0f8e9a666 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Mon, 13 Jul 2026 07:52:59 -0700 Subject: [PATCH 23/34] fix(gpu): omit amdgpu from initrd; multi-GPU init storms dracut-initqueue Reported: node with baked-in DKMS amdgpu (8x MI325X) fails to boot -- drops into dracut-emergency after `dracut-initqueue: Timed out for waiting the udev queue being empty`. Screenshots show amdgpu still emitting per-XCP-partition and per-ring init messages long after boot has given up. Root cause: rebuilding the initrd (default true) pulled the DKMS amdgpu modules under /lib/modules//updates/dkms/ into the initrd. amdgpu then loads *before* switch-root, and the sheer volume of udev events it emits on multi-GPU MI silicon (8 GPUs x 8 XCP partitions + JPEG rings + SMU/PSP/KFD handshakes) keeps systemd-udev-settle from ever seeing an empty queue. initqueue times out (~5 min), boot fails, node reboots, loops. amdgpu is not required to mount rootfs -- NVMe/SATA use their own drivers. Load it *after* switch-root via /etc/modules-load.d/amdgpu.conf (which the script was already writing), where there is no timeout pressure. This matches kairos-a176's observed behavior with the in-tree driver. Two guards: 1. Flip default AMDGPU_REBUILD_INITRD to `false` (was `true`) in Earthfile, .arg.template, and install script. 2. Even when someone sets AMDGPU_REBUILD_INITRD=true, drop /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf with `omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp "` so dracut still leaves amdgpu out. Applied consistently across all three code paths in install-amdgpu-drivers.sh (artifact fast path, inbox mode, in-buildkit DKMS fallback). Co-Authored-By: Claude Opus 4.7 --- .arg.template | 6 ++- Earthfile | 7 +++- scripts/install-amdgpu-drivers.sh | 67 ++++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/.arg.template b/.arg.template index 3c44af61..82d49c6a 100644 --- a/.arg.template +++ b/.arg.template @@ -64,4 +64,8 @@ FORCE_INTERACTIVE_INSTALL=false # # 31.x is tech-preview -- do not mix with a # # production operator. See the version- # # alignment table in docs/amd-gpu-airgapped.md. -# AMDGPU_REBUILD_INITRD=true # Rebuild initrd for the image kernel +# AMDGPU_REBUILD_INITRD=false # Default false. amdgpu is intentionally +# # kept out of the initrd (multi-GPU init +# # emits enough udev events to time out +# # dracut-initqueue). It loads after +# # switch-root via modules-load.d. diff --git a/Earthfile b/Earthfile index a41f3a7a..11a0c83c 100644 --- a/Earthfile +++ b/Earthfile @@ -95,7 +95,12 @@ ARG AMDGPU_DRIVER_RELEASE=7.2.1 # in-buildkit DKMS install (which fails in buildkit's RUN sandbox -- see docs) # and simply extracts the pre-built modules + firmware + config drop-ins. ARG AMDGPU_ARTIFACT_PATH="" -ARG AMDGPU_REBUILD_INITRD=true +# Default false: amdgpu is intentionally omitted from the initrd (see +# scripts/install-amdgpu-drivers.sh -- multi-GPU amdgpu init emits enough +# udev events to blow past dracut-initqueue's udev-settle timeout, dropping +# the node into emergency mode). amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. +ARG AMDGPU_REBUILD_INITRD=false # NVIDIA and AMD driver pre-install are mutually exclusive within a single image. IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] && [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index 7fd364dc..6d2889e8 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -61,7 +61,12 @@ # production operator. Ignored in inbox mode. See # docs/amd-gpu-airgapped.md for the compat matrix. # AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. -# Default: true. +# Default: false. amdgpu is intentionally NOT +# included in the initrd (see the dracut omit +# drop-in the script writes). The base image's +# existing initrd already handles rootfs mount; +# amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf. # set -eo pipefail set -u @@ -75,7 +80,7 @@ die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } # --------------------------------------------------------------------------- AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" -AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-true}" +AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-false}" case "${AMDGPU_DRIVER_SOURCE}" in dkms|inbox) ;; @@ -131,9 +136,28 @@ EOF log "Running depmod -a ${KVER} ..." depmod -a "${KVER}" + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then if command -v dracut >/dev/null 2>&1; then - log "Rebuilding initrd for ${KVER} (dracut) ..." + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." dracut -f "/boot/initrd-${KVER}" "${KVER}" ln -sf "initrd-${KVER}" /boot/initrd elif command -v update-initramfs >/dev/null 2>&1; then @@ -182,9 +206,28 @@ and rebuild (AMDGPU_FORCE_REBUILD=1) or verify BASE_IMAGE matches." log "Running depmod -a ${KVER} ..." depmod -a "${KVER}" || die "depmod failed for ${KVER}." + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then if command -v dracut >/dev/null 2>&1; then - log "Rebuilding initrd for ${KVER} (dracut) ..." + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." dracut -f "/boot/initrd-${KVER}" "${KVER}" ln -sf "initrd-${KVER}" /boot/initrd elif command -v update-initramfs >/dev/null 2>&1; then @@ -447,10 +490,22 @@ log "Running depmod -a ${KVER} ..." depmod -a "${KVER}" || warn "depmod reported an error." # --------------------------------------------------------------------------- -# 10. Rebuild the initrd for the target kernel +# 10. Drop dracut config that OMITS amdgpu from any rebuilt initrd. See the +# equivalent block in the artifact / inbox branches for full rationale -- +# multi-GPU amdgpu init blows past initqueue's timeout when loaded early. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. See install script. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + +# --------------------------------------------------------------------------- +# 11. Rebuild the initrd for the target kernel (amdgpu omitted per above). # --------------------------------------------------------------------------- if [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then - log "Rebuilding initrd for ${KVER} (dracut) ..." + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then ln -sf "initrd-${KVER}" /boot/initrd else From 719cd52d2084a193dddb85278f47b5f238eaa89f Mon Sep 17 00:00:00 2001 From: Nianyu Shen Date: Thu, 16 Jul 2026 08:46:41 -0700 Subject: [PATCH 24/34] feat: support installer boot from USB media on Renesas xHCI hardware (#687) * feat: support installer boot from USB media on Renesas xHCI hardware Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs and disable PCI reallocation on the installer boot entries so the Palette Edge installer can boot and install from USB media on hardware using that chipset. - Earthfile: write /etc/dracut.conf.d/99-usb-media.conf in base-image before the distro dracut regeneration so xhci_pci_renesas is included in the generated initramfs (add_drivers + force_drivers). - grub.cfg: add pci=realloc=off to the installer menu entries. * feat(fips): bundle Renesas xHCI driver into initramfs for USB media boot The Earthfile skips dracut regeneration for FIPS builds, so the USB media driver config must be added in the ubuntu-fips Dockerfiles where kairos-init regenerates the initramfs (-s init --fips). Add a 99-usb-media.conf (xhci_pci_renesas via add_drivers + force_drivers) to each Ubuntu FIPS version (20.04/22.04/24.04) and COPY it into /etc/dracut.conf.d/ before the kairos-init init step that runs dracut. --- Earthfile | 6 ++++++ overlay/files-iso/boot/grub2/grub.cfg | 6 +++--- ubuntu-fips/20.04/99-usb-media.conf | 3 +++ ubuntu-fips/20.04/Dockerfile | 5 +++++ ubuntu-fips/22.04/99-usb-media.conf | 3 +++ ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips | 5 +++++ ubuntu-fips/24.04/99-usb-media.conf | 3 +++ ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips | 5 +++++ 8 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 ubuntu-fips/20.04/99-usb-media.conf create mode 100644 ubuntu-fips/22.04/99-usb-media.conf create mode 100644 ubuntu-fips/24.04/99-usb-media.conf diff --git a/Earthfile b/Earthfile index 11a0c83c..4907c0b3 100644 --- a/Earthfile +++ b/Earthfile @@ -770,6 +770,12 @@ base-image: COPY cloudconfigs/80_stylus_maas.yaml /system/oem/80_stylus_maas.yaml END + # Ensure the Renesas xHCI (USB 3.0) host controller driver is bundled into the + # initramfs so installation from USB media works on hardware using that chipset. + # Must run before the distro dracut regeneration below so the driver is included. + RUN mkdir -p /etc/dracut.conf.d && \ + printf '%s\n' 'hostonly="no"' 'add_drivers+=" xhci_pci_renesas "' 'force_drivers+=" xhci_pci_renesas "' > /etc/dracut.conf.d/99-usb-media.conf + # OS == Ubuntu IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$ARCH" = "amd64" ] IF [ ! -z "$UBUNTU_PRO_KEY" ] diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index 448421e7..b70116d0 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -13,21 +13,21 @@ if [ -f ${font} ];then fi menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off interactive-install echo Loading initrd... $initrd ($root)/boot/initrd } diff --git a/ubuntu-fips/20.04/99-usb-media.conf b/ubuntu-fips/20.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/20.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/20.04/Dockerfile b/ubuntu-fips/20.04/Dockerfile index 2cf14ee2..1c8365ab 100644 --- a/ubuntu-fips/20.04/Dockerfile +++ b/ubuntu-fips/20.04/Dockerfile @@ -112,6 +112,11 @@ RUN cd /usr/lib/dracut/modules.d/95iscsi && patch < /dracut-broken-iscsi-ubuntu- COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | grep fips | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/22.04/99-usb-media.conf b/ubuntu-fips/22.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/22.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips index 5d4c06be..282bc8a3 100644 --- a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips +++ b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips @@ -32,6 +32,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 22.04/dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 22.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY 22.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/24.04/99-usb-media.conf b/ubuntu-fips/24.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/24.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips index a9d1d6de..f8e63381 100644 --- a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips +++ b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips @@ -41,6 +41,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 24.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 24.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + COPY 24.04/fix.sh /tmp/fix.sh COPY stig-remediate.sh /tmp/stig-remediate.sh COPY restore-ubuntu-default-banners.sh /tmp/restore-ubuntu-default-banners.sh From 83cfcf6edf1d8a1b3b1ab94ec98461af2d232f79 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 16 Jul 2026 21:26:20 -0700 Subject: [PATCH 25/34] fix(boot): block nouveau + qat_4xxx on installer ISO and installed OS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installer ISO was hanging on the same NVIDIA data-center GPUs previously addressed for the installed OS in 1c3a089: nouveau auto-loads in the live ISO's initramfs before switchroot, hits GSP init, and stalls udev-settle indefinitely. Runtime bootargs.cfg had the fix; installer ISO did not. Also block qat_4xxx unconditionally: Xeon Scalable 4th/5th gen hosts with Intel QuickAssist devices load qat_4xxx in initramfs and its probe / firmware-load path adds minutes to boot. CanvOS does not consume QAT acceleration, so blacklisting is safe on every build. Three sites, one arg pair each (rd.driver.blacklist= is the load-bearing flag — dracut honors it before udev fires; modprobe.blacklist= is belt-and-braces for post-switchroot): * overlay/files-iso/boot/grub2/grub.cfg: append rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 to all three menuentries (default install, manual install, interactive install). Unconditional — the same ISO installs onto NVIDIA and non-NVIDIA hardware, and blacklisting an absent module is a no-op. * Earthfile CMDLINE default: same three args appended, so enki --extend-cmdline bakes them into the UKI stub for both signed and unsigned installer paths. Addresses the follow-up flagged in 1c3a089 for the UKI build path. * Earthfile base-image bootargs.cfg block: new unconditional RUN for qat_4xxx. Two-branch sed — if the nouveau block above already added rd.driver.blacklist=, splice ,qat_4xxx onto the existing list rather than duplicating the arg key (some cmdline parsers treat repeated keys as last-wins and silently drop the earlier value). Otherwise add a fresh rd.driver.blacklist=qat_4xxx modprobe.blacklist=qat_4xxx pair. Idempotent via outer grep guard. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 17 ++++++++++++++++- overlay/files-iso/boot/grub2/grub.cfg | 6 +++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Earthfile b/Earthfile index 4907c0b3..eb8c3165 100644 --- a/Earthfile +++ b/Earthfile @@ -129,7 +129,7 @@ ARG INCLUDE_MS_SECUREBOOT_KEYS=true ARG AUTO_ENROLL_SECUREBOOT_KEYS=false ARG UKI_BRING_YOUR_OWN_KEYS=false -ARG CMDLINE="stylus.registration" +ARG CMDLINE="stylus.registration rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0" ARG BRANDING="Palette eXtended Kubernetes Edge" ARG FORCE_INTERACTIVE_INSTALL=false @@ -976,6 +976,21 @@ base-image: sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau modprobe.blacklist=nouveau nouveau.modeset=0"|' /etc/cos/bootargs.cfg; \ fi END + + # Block Intel QAT 4xxx driver at the kernel command line. On Xeon + # Scalable 4th/5th gen hosts with QAT devices, udev auto-loads + # qat_4xxx in initramfs and its probe/firmware-load stalls boot for + # minutes. rd.driver.blacklist= is the load-bearing flag (dracut + # honors it before udev fires); modprobe.blacklist= is belt-and-braces + # for post-switchroot. Applied unconditionally — CanvOS does not + # consume QAT acceleration. + RUN if ! grep -Fq "rd.driver.blacklist=qat_4xxx" /etc/cos/bootargs.cfg; then \ + if grep -Fq "rd.driver.blacklist=" /etc/cos/bootargs.cfg; then \ + sed -i 's|\(rd\.driver\.blacklist=[^ "]*\)|\1,qat_4xxx|; s|\(modprobe\.blacklist=[^ "]*\)|\1,qat_4xxx|' /etc/cos/bootargs.cfg; \ + else \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=qat_4xxx modprobe.blacklist=qat_4xxx"|' /etc/cos/bootargs.cfg; \ + fi \ + fi END KAIROS_RELEASE: diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index b70116d0..a3bb3162 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -13,21 +13,21 @@ if [ -f ${font} ];then fi menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install echo Loading initrd... $initrd ($root)/boot/initrd } From fd06a58b75cef1b528cf33d768c0e67e2852272a Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Fri, 17 Jul 2026 03:07:27 -0700 Subject: [PATCH 26/34] feat(gpu): enable NVIDIA fabricmanager + nscq + imex by default for HGX/DGX/GB200 HGX H100/H200, HGX B200, DGX and GB200 need nvidia-fabricmanager to bring up multi-GPU NVLink; GB200 NVL72 additionally needs nvidia-imex for multi-node NVLink Sharp. Both daemons are userspace, have no kernel side effect, and exit cleanly on hardware that does not need them (fabricmanager: "No NvSwitch found" -> failed/inactive; imex: no nodes_config.cfg -> exits, unit inactive), so default-on is safe on non-target fleets at a ~70-110 MB image cost. Also install libnvidia-nscq- explicitly alongside fabricmanager -- it is a transitive dep today, listed explicitly so the build fails loudly if the CUDA repo ever drops the auto-dep. IMEX is best-effort: pre-570 driver branches do not publish the package and the install step warns and skips. Opt-out per node type via --NVIDIA_INSTALL_FABRICMANAGER=false / --NVIDIA_INSTALL_IMEX=false. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 4 ++- docs/nvidia-gpu-airgapped.md | 3 +- scripts/install-nvidia-drivers.sh | 58 +++++++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/Earthfile b/Earthfile index eb8c3165..aa6ee331 100644 --- a/Earthfile +++ b/Earthfile @@ -70,7 +70,8 @@ ARG INSTALL_NVIDIA_GPU_DRIVERS=false ARG NVIDIA_DRIVER_BRANCH=580 ARG NVIDIA_DRIVER_TYPE=open ARG NVIDIA_USE_CUDA_REPO=true -ARG NVIDIA_INSTALL_FABRICMANAGER=false +ARG NVIDIA_INSTALL_FABRICMANAGER=true +ARG NVIDIA_INSTALL_IMEX=true ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false ARG NVIDIA_REBUILD_INITRD=true @@ -859,6 +860,7 @@ base-image: NVIDIA_DRIVER_TYPE="$NVIDIA_DRIVER_TYPE" \ NVIDIA_USE_CUDA_REPO="$NVIDIA_USE_CUDA_REPO" \ NVIDIA_INSTALL_FABRICMANAGER="$NVIDIA_INSTALL_FABRICMANAGER" \ + NVIDIA_INSTALL_IMEX="$NVIDIA_INSTALL_IMEX" \ NVIDIA_INSTALL_CONTAINER_TOOLKIT="$NVIDIA_INSTALL_CONTAINER_TOOLKIT" \ NVIDIA_REBUILD_INITRD="$NVIDIA_REBUILD_INITRD" \ /tmp/install-nvidia-drivers.sh && \ diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index dd462068..aaf4d754 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -157,7 +157,8 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | -| `NVIDIA_INSTALL_FABRICMANAGER` | `false` | Set `true` for NVSwitch / HGX systems (installs and enables `nvidia-fabricmanager`). | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. On non-NVSwitch hosts the daemon exits early and the unit stays inactive; no kernel side effect, no restart loop, ~60–90 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | | `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | | `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 70e4d596..78bcf83f 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -17,7 +17,9 @@ # * the NVIDIA kernel modules (nvidia, nvidia-uvm, nvidia-modeset, # nvidia-drm, nvidia-peermem) built with DKMS against the IMAGE kernel # * nouveau blacklist + nvidia module autoload + nvidia-persistenced -# * (optional) nvidia-fabricmanager for HGX / NVSwitch systems +# * nvidia-fabricmanager + libnvidia-nscq for HGX / NVSwitch systems +# (HGX H100/H200, HGX B200, DGX, GB200) -- required for multi-GPU NVLink +# * nvidia-imex for GB200 NVL72 multi-node NVLink Sharp (Blackwell) # # WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, # deployed by the GPU Operator itself): @@ -57,7 +59,22 @@ # repo (recommended, has every -server branch). # "false" to use only Ubuntu's own repos. # Default: true -# NVIDIA_INSTALL_FABRICMANAGER "true" for NVSwitch/HGX boxes. Default: false +# NVIDIA_INSTALL_FABRICMANAGER "true" for NVSwitch/HGX boxes (H100/B200 HGX, +# DGX, GB200). Default: true. Harmless on non- +# NVSwitch hosts: the unit exits early and stays +# inactive; no restart loop, no kernel effect, +# ~60-90 MB image cost. Also installs the +# matching libnvidia-nscq- explicitly. +# See docs/nvidia-gpu-airgapped.md. +# NVIDIA_INSTALL_IMEX "true" to also install nvidia-imex-, +# the Internode Memory Exchange daemon required +# for GB200 NVL72 multi-node NVLink Sharp +# (Blackwell, driver 570+). Default: true. +# Harmless on non-NVL72 boxes: without +# /etc/nvidia-imex/nodes_config.cfg the daemon +# exits and the unit stays inactive. Best-effort +# -- skipped with a warning on branches that +# predate IMEX (pre-570). # NVIDIA_INSTALL_CONTAINER_TOOLKIT "true" to ALSO pre-install # nvidia-container-toolkit on the host (then set # toolkit.enabled=false in the operator). @@ -78,7 +95,8 @@ die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-580}" NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-open}" NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" -NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-false}" +NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-true}" +NVIDIA_INSTALL_IMEX="${NVIDIA_INSTALL_IMEX:-true}" NVIDIA_INSTALL_CONTAINER_TOOLKIT="${NVIDIA_INSTALL_CONTAINER_TOOLKIT:-false}" NVIDIA_REBUILD_INITRD="${NVIDIA_REBUILD_INITRD:-true}" @@ -238,15 +256,41 @@ matching gcc are installed." fi # --------------------------------------------------------------------------- -# 8. Optional: NVIDIA Fabric Manager (NVSwitch / HGX systems only) +# 8. NVIDIA Fabric Manager + libnvidia-nscq (HGX / NVSwitch systems) +# Required on HGX H100/H200, HGX B200, DGX, GB200 for multi-GPU NVLink. +# libnvidia-nscq- is a fabricmanager dep and gets pulled in +# transitively -- listed explicitly so the install fails loudly if the +# CUDA repo ever drops the auto-dep. Harmless on non-NVSwitch hosts: +# the daemon exits ("No NvSwitch found") and the unit stays failed with +# no kernel side effect and no restart loop. # --------------------------------------------------------------------------- if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" - log "Installing ${FM_PKG} ..." - if apt-get install -y --no-install-recommends "${FM_PKG}"; then + NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${FM_PKG} + ${NSCQ_PKG} ..." + if apt-get install -y --no-install-recommends "${FM_PKG}" "${NSCQ_PKG}"; then systemctl enable nvidia-fabricmanager.service 2>/dev/null || true else - warn "could not install ${FM_PKG}; skipping fabric manager." + warn "could not install ${FM_PKG} / ${NSCQ_PKG}; skipping fabric manager." + fi +fi + +# --------------------------------------------------------------------------- +# 8b. NVIDIA IMEX -- Internode Memory Exchange daemon +# Required for GB200 NVL72 multi-node NVLink Sharp (Blackwell, driver +# 570+). Not needed for single-node HGX B200 or HGX H100. Package is +# part of the CUDA repo; older driver branches (pre-570) do not publish +# it, so this is best-effort. On single-node boxes the daemon has no +# /etc/nvidia-imex/nodes_config.cfg and exits cleanly -- unit stays +# inactive, ~10-20 MB image cost. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_IMEX}" = "true" ]; then + IMEX_PKG="nvidia-imex-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${IMEX_PKG} (GB200 NVL72 multi-node NVLink Sharp) ..." + if apt-get install -y --no-install-recommends "${IMEX_PKG}"; then + systemctl enable nvidia-imex.service 2>/dev/null || true + else + warn "${IMEX_PKG} not available (branch ${NVIDIA_DRIVER_BRANCH} may predate IMEX -- 570+ only). Skipping." fi fi From b764b30c194ea6b9a5da0a0c68e173bae664db0d Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Fri, 17 Jul 2026 09:55:54 -0700 Subject: [PATCH 27/34] fix(gpu): install infiniband-diags with fabricmanager so ibstat is present The nvidia-fabricmanager- package ships a unit whose ExecStart is a wrapper (/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh) that NVIDIA added to gate FM startup on the IB fabric being up -- needed for GB200 NVL72 multi-node NVLink Sharp. The wrapper shells out to `ibstat` (from infiniband-diags) to probe IB device state; without that binary the unit fails immediately on every boot, regardless of hardware, with: "ibstat" command not found! Please install "ibstat". nvidia-fabricmanager.service: Control process exited, code=exited, status=1 Install infiniband-diags whenever fabricmanager is installed. ~5-8 MB, no runtime side effect on non-IB hosts. On non-NVSwitch hardware the wrapper now proceeds to nv-fabricmanager which exits "No NvSwitch found" as originally expected -- the unit stays failed with no kernel effect and no restart loop. Co-Authored-By: Claude Opus 4.7 --- docs/nvidia-gpu-airgapped.md | 2 +- scripts/install-nvidia-drivers.sh | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index aaf4d754..d9a1458a 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -157,7 +157,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | -| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. On non-NVSwitch hosts the daemon exits early and the unit stays inactive; no kernel side effect, no restart loop, ~60–90 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `infiniband-diags` (for `ibstat`, which NVIDIA's shipped unit wrapper `/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh` requires) and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. On non-NVSwitch hosts the daemon exits early and the unit stays inactive; no kernel side effect, no restart loop, ~65–100 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | | `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | | `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | | `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 78bcf83f..c8e3c6f2 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -260,18 +260,28 @@ fi # Required on HGX H100/H200, HGX B200, DGX, GB200 for multi-GPU NVLink. # libnvidia-nscq- is a fabricmanager dep and gets pulled in # transitively -- listed explicitly so the install fails loudly if the -# CUDA repo ever drops the auto-dep. Harmless on non-NVSwitch hosts: -# the daemon exits ("No NvSwitch found") and the unit stays failed with -# no kernel side effect and no restart loop. +# CUDA repo ever drops the auto-dep. +# +# infiniband-diags provides `ibstat`, which NVIDIA's shipped unit wrapper +# /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh calls to +# gate FM startup on the IB fabric being up (needed for GB200 NVL72 +# multi-node NVLink Sharp; the wrapper is present in ALL FM packages +# now). Without ibstat the unit fails immediately with 'ibstat command +# not found', regardless of hardware -- so we install infiniband-diags +# whenever FM is installed. ~5-8 MB, no runtime side effect. +# +# On non-NVSwitch hosts FM still exits ("No NvSwitch found") and the +# unit stays failed with no kernel side effect and no restart loop. # --------------------------------------------------------------------------- if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" - log "Installing ${FM_PKG} + ${NSCQ_PKG} ..." - if apt-get install -y --no-install-recommends "${FM_PKG}" "${NSCQ_PKG}"; then + log "Installing ${FM_PKG} + ${NSCQ_PKG} + infiniband-diags (for ibstat) ..." + if apt-get install -y --no-install-recommends \ + "${FM_PKG}" "${NSCQ_PKG}" infiniband-diags; then systemctl enable nvidia-fabricmanager.service 2>/dev/null || true else - warn "could not install ${FM_PKG} / ${NSCQ_PKG}; skipping fabric manager." + warn "could not install ${FM_PKG} / ${NSCQ_PKG} / infiniband-diags; skipping fabric manager." fi fi From d76ab4d11d01876d28e14b7b43964ca04f9b4bf0 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Fri, 17 Jul 2026 10:22:38 -0700 Subject: [PATCH 28/34] fix(gpu): bypass GB200 NVL72 wrapper on fabricmanager so FM starts on HGX/DGX NVIDIA's 570+ nvidia-fabricmanager- unit invokes a wrapper -- /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh -- that gates FM startup on the GB200 NVL72 NVLink subnet coming up. The wrapper shells out to `ibstat` (infiniband-diags) and `nvlsm` (NVIDIA Subnet Manager, shipped in UFM/MLNX-OFED, not in this package). Installing infiniband-diags gets past the first gate but the wrapper then dies on missing nvlsm, on every non-NVL72 topology (standalone HGX H100/B200, single-node DGX, workstations). FM never runs on any of that hardware. Install a systemd drop-in at /etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf that overrides ExecStart back to calling nv-fabricmanager directly (the pre-wrapper behavior). FM then correctly initializes NVSwitch on HGX/DGX, and cleanly exits "No NvSwitch found" on non-NVSwitch hosts. GB200 NVL72 deployments must remove this drop-in via user-data so the wrapper's IB/nvlsm precondition applies. infiniband-diags is retained defensively (~5-8 MB) so a user reverting to the wrapper doesn't re-encounter the ibstat gate. Co-Authored-By: Claude Opus 4.7 --- docs/nvidia-gpu-airgapped.md | 2 +- scripts/install-nvidia-drivers.sh | 48 ++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index d9a1458a..9164facd 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -157,7 +157,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | -| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `infiniband-diags` (for `ibstat`, which NVIDIA's shipped unit wrapper `/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh` requires) and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. On non-NVSwitch hosts the daemon exits early and the unit stays inactive; no kernel side effect, no restart loop, ~65–100 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `infiniband-diags` and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. Also installs a systemd drop-in at `/etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf` that overrides `ExecStart` to invoke `nv-fabricmanager` directly instead of NVIDIA's shipped `nvidia-fabricmanager-start.sh` wrapper (the wrapper probes for `ibstat` + `nvlsm` to gate on the GB200 NVL72 NVLink subnet, which breaks FM startup on every non-NVL72 topology). **GB200 NVL72 deployments must remove this drop-in via user-data** so the wrapper's IB/nvlsm precondition applies. On non-NVSwitch hosts the daemon exits `"No NvSwitch found"` and the unit stays inactive; no kernel side effect, no restart loop, ~65–100 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | | `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | | `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | | `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index c8e3c6f2..92229035 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -262,23 +262,51 @@ fi # transitively -- listed explicitly so the install fails loudly if the # CUDA repo ever drops the auto-dep. # -# infiniband-diags provides `ibstat`, which NVIDIA's shipped unit wrapper -# /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh calls to -# gate FM startup on the IB fabric being up (needed for GB200 NVL72 -# multi-node NVLink Sharp; the wrapper is present in ALL FM packages -# now). Without ibstat the unit fails immediately with 'ibstat command -# not found', regardless of hardware -- so we install infiniband-diags -# whenever FM is installed. ~5-8 MB, no runtime side effect. +# Wrapper bypass. NVIDIA 570+ packages ship a systemd unit whose +# ExecStart is /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh, +# a wrapper added to gate FM startup on the GB200 NVL72 NVLink subnet +# coming up: it shells out to `ibstat` (infiniband-diags) and `nvlsm` +# (the NVIDIA Subnet Manager, shipped separately in UFM/MLNX-OFED, not +# in this package). On every non-NVL72 host -- standalone HGX H100/B200, +# single-node DGX, workstations, dev boxes -- one of those deps is +# absent and the unit fails with either: +# "ibstat command not found! Please install ibstat." +# or a similar 'nvlsm not found' error, before nv-fabricmanager is +# ever invoked. That breaks FM on the entire non-NVL72 fleet. # -# On non-NVSwitch hosts FM still exits ("No NvSwitch found") and the -# unit stays failed with no kernel side effect and no restart loop. +# Fix: install a drop-in that overrides ExecStart back to calling +# nv-fabricmanager directly (pre-wrapper behavior). FM then correctly +# initializes NVSwitch on HGX/DGX, and cleanly exits "No NvSwitch found" +# on non-NVSwitch hosts. +# +# GB200 NVL72 caveat: the drop-in also removes the IB/nvlsm precondition +# that NVL72 legitimately needs. Deployments targeting NVL72 should +# remove /etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf +# via user-data and ensure ibstat + nvlsm are provisioned. +# +# infiniband-diags is still installed defensively (~5-8 MB): the drop-in +# is easy to override and having ibstat present means the vendor wrapper +# at least clears its first gate if a user reverts to it. # --------------------------------------------------------------------------- if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" - log "Installing ${FM_PKG} + ${NSCQ_PKG} + infiniband-diags (for ibstat) ..." + log "Installing ${FM_PKG} + ${NSCQ_PKG} + infiniband-diags ..." if apt-get install -y --no-install-recommends \ "${FM_PKG}" "${NSCQ_PKG}" infiniband-diags; then + log "Installing systemd drop-in to bypass GB200 NVL72 wrapper (ibstat/nvlsm gate) ..." + mkdir -p /etc/systemd/system/nvidia-fabricmanager.service.d + cat > /etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# NVIDIA 570+ FM packages wrap ExecStart in nvidia-fabricmanager-start.sh, +# which probes for ibstat + nvlsm to gate startup on the GB200 NVL72 NVLink +# subnet. On non-NVL72 hardware nvlsm is absent and the wrapper fails before +# nv-fabricmanager is ever invoked. Bypass it and call the daemon directly. +# GB200 NVL72 deployments should remove this drop-in via user-data. +[Service] +ExecStart= +ExecStart=/usr/bin/nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg +EOF systemctl enable nvidia-fabricmanager.service 2>/dev/null || true else warn "could not install ${FM_PKG} / ${NSCQ_PKG} / infiniband-diags; skipping fabric manager." From 54a2864a2df279e644f064a35c77b4c22fc74ac2 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Fri, 17 Jul 2026 10:33:59 -0700 Subject: [PATCH 29/34] fix(gpu): install nvlsm with fabricmanager so vendor wrapper works on all hardware The NVIDIA 570+ fabricmanager unit invokes a wrapper (/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh) that probes `ibstat` (infiniband-diags) and `nvlsm` (NVIDIA Subnet Manager) before starting nv-fabricmanager -- needed to gate on the GB200 NVL72 NVLink subnet coming up. Without both binaries the unit dies before FM is ever invoked. Prior fix bypassed the wrapper via a systemd drop-in. That worked, but diverges from the vendor path and would break GB200 NVL72 deployments that legitimately need the IB/nvlsm precondition. Verified on a live HGX host that installing nvlsm (2025.10.14-1, from the CUDA repo we already add -- unversioned package name) lets the vendor wrapper succeed on standalone HGX topologies too. FM comes up and GPU Operator pods reach Ready. Install nvlsm alongside fabricmanager + libnvidia-nscq + infiniband-diags, and drop the 10-bypass-wrapper.conf systemd override. Same vendor code path on HGX/DGX and GB200 NVL72 alike. Co-Authored-By: Claude Opus 4.7 --- docs/nvidia-gpu-airgapped.md | 2 +- scripts/install-nvidia-drivers.sh | 58 ++++++++++--------------------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md index 9164facd..a802230f 100644 --- a/docs/nvidia-gpu-airgapped.md +++ b/docs/nvidia-gpu-airgapped.md @@ -157,7 +157,7 @@ All variables are optional and have defaults. Set them in `.arg` or pass as | `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | | `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | | `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | -| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `infiniband-diags` and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. Also installs a systemd drop-in at `/etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf` that overrides `ExecStart` to invoke `nv-fabricmanager` directly instead of NVIDIA's shipped `nvidia-fabricmanager-start.sh` wrapper (the wrapper probes for `ibstat` + `nvlsm` to gate on the GB200 NVL72 NVLink subnet, which breaks FM startup on every non-NVL72 topology). **GB200 NVL72 deployments must remove this drop-in via user-data** so the wrapper's IB/nvlsm precondition applies. On non-NVSwitch hosts the daemon exits `"No NvSwitch found"` and the unit stays inactive; no kernel side effect, no restart loop, ~65–100 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `nvlsm` (NVIDIA Subnet Manager, from the CUDA repo) + `infiniband-diags` (for `ibstat`) and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. NVIDIA's 570+ shipped unit invokes a wrapper (`/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh`) that probes `ibstat` + `nvlsm` before starting `nv-fabricmanager`; all four packages must be present or the unit fails before FM is ever invoked. On non-NVSwitch hosts FM exits `"No NvSwitch found"` and the unit stays inactive; no kernel side effect, no restart loop, ~70–110 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | | `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | | `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | | `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 92229035..046c4885 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -256,60 +256,38 @@ matching gcc are installed." fi # --------------------------------------------------------------------------- -# 8. NVIDIA Fabric Manager + libnvidia-nscq (HGX / NVSwitch systems) +# 8. NVIDIA Fabric Manager + libnvidia-nscq + nvlsm (HGX / NVSwitch systems) # Required on HGX H100/H200, HGX B200, DGX, GB200 for multi-GPU NVLink. # libnvidia-nscq- is a fabricmanager dep and gets pulled in # transitively -- listed explicitly so the install fails loudly if the # CUDA repo ever drops the auto-dep. # -# Wrapper bypass. NVIDIA 570+ packages ship a systemd unit whose -# ExecStart is /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh, -# a wrapper added to gate FM startup on the GB200 NVL72 NVLink subnet -# coming up: it shells out to `ibstat` (infiniband-diags) and `nvlsm` -# (the NVIDIA Subnet Manager, shipped separately in UFM/MLNX-OFED, not -# in this package). On every non-NVL72 host -- standalone HGX H100/B200, -# single-node DGX, workstations, dev boxes -- one of those deps is -# absent and the unit fails with either: +# NVIDIA 570+ ships the FM unit with ExecStart wrapped in +# /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh, which +# probes `ibstat` (infiniband-diags) and `nvlsm` (NVIDIA Subnet Manager) +# before invoking nv-fabricmanager -- needed so the NVLink subnet is up +# on GB200 NVL72. Both binaries must be present or the unit dies: # "ibstat command not found! Please install ibstat." -# or a similar 'nvlsm not found' error, before nv-fabricmanager is -# ever invoked. That breaks FM on the entire non-NVL72 fleet. +# "nvlsm command not found! Please install nvlsm." +# Empirically the wrapper succeeds on standalone HGX topologies once +# both are installed (verified on HGX with nvlsm 2025.10.14-1 from the +# NVIDIA CUDA repo we already added in step 4), so we install them +# alongside FM and let the vendor wrapper stay in charge. # -# Fix: install a drop-in that overrides ExecStart back to calling -# nv-fabricmanager directly (pre-wrapper behavior). FM then correctly -# initializes NVSwitch on HGX/DGX, and cleanly exits "No NvSwitch found" -# on non-NVSwitch hosts. -# -# GB200 NVL72 caveat: the drop-in also removes the IB/nvlsm precondition -# that NVL72 legitimately needs. Deployments targeting NVL72 should -# remove /etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf -# via user-data and ensure ibstat + nvlsm are provisioned. -# -# infiniband-diags is still installed defensively (~5-8 MB): the drop-in -# is easy to override and having ibstat present means the vendor wrapper -# at least clears its first gate if a user reverts to it. +# Package sources: nvlsm ships in the CUDA repo under an unversioned +# package name (not nvlsm-). infiniband-diags is Ubuntu-native. +# On non-NVSwitch hosts nv-fabricmanager still exits "No NvSwitch found" +# and the unit stays inactive; no kernel side effect, no restart loop. # --------------------------------------------------------------------------- if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" - log "Installing ${FM_PKG} + ${NSCQ_PKG} + infiniband-diags ..." + log "Installing ${FM_PKG} + ${NSCQ_PKG} + nvlsm + infiniband-diags ..." if apt-get install -y --no-install-recommends \ - "${FM_PKG}" "${NSCQ_PKG}" infiniband-diags; then - log "Installing systemd drop-in to bypass GB200 NVL72 wrapper (ibstat/nvlsm gate) ..." - mkdir -p /etc/systemd/system/nvidia-fabricmanager.service.d - cat > /etc/systemd/system/nvidia-fabricmanager.service.d/10-bypass-wrapper.conf <<'EOF' -# Managed by CanvOS install-nvidia-drivers.sh -# NVIDIA 570+ FM packages wrap ExecStart in nvidia-fabricmanager-start.sh, -# which probes for ibstat + nvlsm to gate startup on the GB200 NVL72 NVLink -# subnet. On non-NVL72 hardware nvlsm is absent and the wrapper fails before -# nv-fabricmanager is ever invoked. Bypass it and call the daemon directly. -# GB200 NVL72 deployments should remove this drop-in via user-data. -[Service] -ExecStart= -ExecStart=/usr/bin/nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg -EOF + "${FM_PKG}" "${NSCQ_PKG}" nvlsm infiniband-diags; then systemctl enable nvidia-fabricmanager.service 2>/dev/null || true else - warn "could not install ${FM_PKG} / ${NSCQ_PKG} / infiniband-diags; skipping fabric manager." + warn "could not install ${FM_PKG} / ${NSCQ_PKG} / nvlsm / infiniband-diags; skipping fabric manager." fi fi From 8ca2f16dde49bb435690a8e63cfec0d3f04bb2f1 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Sat, 18 Jul 2026 18:06:19 -0700 Subject: [PATCH 30/34] fix(boot): apply nouveau + qat_4xxx + pci=realloc=off to every installed OS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 83cfcf6. The previous nouveau block was gated on INSTALL_NVIDIA_GPU_DRIVERS=true (inherited from 1c3a089), which meant non-NVIDIA CanvOS builds that later landed on NVIDIA hardware still hung in GSP init on first boot. The image doesn't know its target hardware at build time — the hang is a property of the host, not the build flag — so blacklisting must be unconditional. Same reasoning as qat_4xxx, which was already unconditional in the same block. Collapses the two prior sed passes (gated nouveau + unconditional qat_4xxx-splice) into one unconditional sed that appends both blacklists plus pci=realloc=off in a single append, mirroring the installer ISO grub.cfg cmdline. Idempotent via the grep guard on "rd.driver.blacklist=nouveau". pci=realloc=off pins the PCI BAR layout to firmware assignments; kernel-side reallocation has caused resource conflicts on some server platforms, and the installer ISO already carries it. UKI CMDLINE default reverted — UKI installer path is out of scope for this change per user direction. Co-Authored-By: Claude Opus 4.7 --- Earthfile | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/Earthfile b/Earthfile index aa6ee331..afbb22a6 100644 --- a/Earthfile +++ b/Earthfile @@ -130,7 +130,7 @@ ARG INCLUDE_MS_SECUREBOOT_KEYS=true ARG AUTO_ENROLL_SECUREBOOT_KEYS=false ARG UKI_BRING_YOUR_OWN_KEYS=false -ARG CMDLINE="stylus.registration rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0" +ARG CMDLINE="stylus.registration" ARG BRANDING="Palette eXtended Kubernetes Edge" ARG FORCE_INTERACTIVE_INSTALL=false @@ -968,30 +968,27 @@ base-image: sed -i 's|\(set baseCmd="[^"]*\)"|\1 systemd.unified_cgroup_hierarchy=1"|' /etc/cos/bootargs.cfg; \ fi - # When the NVIDIA driver is pre-installed, block nouveau at the kernel - # command line. modprobe.d blacklists don't apply until AFTER switchroot, - # by which point initramfs udev has already auto-loaded nouveau on - # modern data-center GPUs (Ada/Hopper/Blackwell) and hung in GSP init, - # stalling systemd-udev-settle indefinitely. - IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] - RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ - sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau modprobe.blacklist=nouveau nouveau.modeset=0"|' /etc/cos/bootargs.cfg; \ - fi - END - - # Block Intel QAT 4xxx driver at the kernel command line. On Xeon - # Scalable 4th/5th gen hosts with QAT devices, udev auto-loads - # qat_4xxx in initramfs and its probe/firmware-load stalls boot for - # minutes. rd.driver.blacklist= is the load-bearing flag (dracut - # honors it before udev fires); modprobe.blacklist= is belt-and-braces - # for post-switchroot. Applied unconditionally — CanvOS does not - # consume QAT acceleration. - RUN if ! grep -Fq "rd.driver.blacklist=qat_4xxx" /etc/cos/bootargs.cfg; then \ - if grep -Fq "rd.driver.blacklist=" /etc/cos/bootargs.cfg; then \ - sed -i 's|\(rd\.driver\.blacklist=[^ "]*\)|\1,qat_4xxx|; s|\(modprobe\.blacklist=[^ "]*\)|\1,qat_4xxx|' /etc/cos/bootargs.cfg; \ - else \ - sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=qat_4xxx modprobe.blacklist=qat_4xxx"|' /etc/cos/bootargs.cfg; \ - fi \ + # Block nouveau and qat_4xxx at the kernel command line on every + # build, and pin PCI BAR layout to firmware assignments. + # + # nouveau: modern NVIDIA data-center GPUs (Ada/Hopper/Blackwell) + # hang in GSP init when initramfs udev auto-loads nouveau before + # switchroot, stalling systemd-udev-settle indefinitely. Applied + # unconditionally — the image may be installed onto NVIDIA hardware + # even when INSTALL_NVIDIA_GPU_DRIVERS=false. rd.driver.blacklist= + # is the load-bearing flag (dracut honors it before udev fires); + # modprobe.blacklist= is belt-and-braces for post-switchroot. + # Harmless when no NVIDIA GPU is present. + # + # qat_4xxx: on Xeon Scalable 4th/5th gen hosts with QAT devices, + # udev auto-loads qat_4xxx in initramfs and its probe/firmware-load + # stalls boot for minutes. CanvOS does not consume QAT acceleration. + # + # pci=realloc=off: firmware-assigned PCI resource layout is + # authoritative; kernel-side reallocation has caused BAR conflicts + # on some server platforms. Mirrors the installer ISO cmdline. + RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 pci=realloc=off"|' /etc/cos/bootargs.cfg; \ fi END From 04b1f7da0d7806c7ee303c98c88b612189e4057c Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Mon, 20 Jul 2026 23:02:35 +0530 Subject: [PATCH 31/34] fix(gpu): autoload ib_umad so fabricmanager precheck passes on NVL5+ (HGX B200) (#700) nvidia-fabricmanager 570+ ships a systemd unit whose ExecStartPre invokes `nvidia-fabricmanager-start.sh --mode precheck`. On HGX B200 / GB200 hardware the wrapper takes the "Detected NVL5+ system" branch, which requires the ib_umad kernel module to be loaded before nv-fabricmanager can start -- the daemon opens /dev/infiniband/umad* character devices to send MADs to the NVSwitch fabric (NVSwitch reuses the InfiniBand management-datagram shape for topology discovery + NVLink route programming). Kairos edge images ship the module in the kernel-modules-extra set but don't auto-load it. Result: on first boot of a Palette Edge HGX B200 appliance, the service fails at ExecStartPre with: Detected NVL5+ system Kernel module "ib_umad" has not been loaded, fabric manager cannot be started Please run "modprobe ib_umad" before starting fabric manager That cascades: without fabricmanager the NVSwitch fabric never initializes; CUDA runtime in any container fails cudaMalloc with CUDA_ERROR_NOT_INITIALIZED ("system not yet initialized"); nvidia-cuda-validator crashes at Init:Error; nvidia-operator-validator hangs at Init:2/4. Fix: write /etc/modules-load.d/nvidia-fabricmanager.conf listing ib_umad so systemd's kmod-static-nodes / systemd-modules-load loads it at boot ahead of nvidia-fabricmanager.service. The file is scoped to the same conditional that installs the fabricmanager package (NVIDIA_INSTALL_FABRICMANAGER=true), so hardware where the operator opts out of fabricmanager doesn't get the module load either. On non-NVSwitch hosts the module costs ~10 KiB of RSS and has no other effect; fabricmanager still exits "No NvSwitch found" and the unit stays inactive as before. Verified live on 8x HGX B200 running driver 580.159.03: with the drop-in in place, `systemctl start nvidia-fabricmanager` transitions from failed to active in ~3s, the NVSwitch fabric initializes, and downstream nvidia-cuda-validator exits 0 on the next GPU-operator reconcile. Co-authored-by: Claude Opus 4.7 --- scripts/install-nvidia-drivers.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh index 046c4885..0acde144 100755 --- a/scripts/install-nvidia-drivers.sh +++ b/scripts/install-nvidia-drivers.sh @@ -286,6 +286,36 @@ if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then if apt-get install -y --no-install-recommends \ "${FM_PKG}" "${NSCQ_PKG}" nvlsm infiniband-diags; then systemctl enable nvidia-fabricmanager.service 2>/dev/null || true + + # --- ib_umad autoload for fabricmanager precheck -------------------- + # NVIDIA 570+ nvidia-fabricmanager-start.sh --mode precheck (invoked + # by the systemd unit's ExecStartPre) takes the "Detected NVL5+ + # system" branch on HGX B200 / GB200 hardware and requires the + # ib_umad kernel module to be loaded before nv-fabricmanager can + # start -- the wrapper opens /dev/infiniband/umad* to send MADs to + # the NVSwitch fabric (NVSwitch reuses the InfiniBand management- + # datagram shape). Kairos edge images ship the module (it's in the + # kernel-modules-extra set) but don't auto-load it, so precheck + # dies with: + # Detected NVL5+ system + # Kernel module "ib_umad" has not been loaded, + # fabric manager cannot be started + # Please run "modprobe ib_umad" before starting fabric manager + # Load it at boot so the service comes up on NVSwitch hardware + # without any operator intervention. Verified live on 8x HGX B200 + # (driver 580.159.03): with this in place nvidia-fabricmanager.service + # goes active in ~3s at boot and downstream nvidia-cuda-validator + # exits 0 on the next GPU-operator reconcile. + # + # On non-NVSwitch hosts the module load costs ~10 KiB of RSS and has + # no other side effect; fabricmanager still exits "No NvSwitch found" + # and the unit stays inactive as before. + cat > /etc/modules-load.d/nvidia-fabricmanager.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Required by nvidia-fabricmanager-start.sh --mode precheck on NVL5+ +# systems (HGX B200, GB200) -- see /usr/bin/nvidia-fabricmanager-start.sh. +ib_umad +EOF else warn "could not install ${FM_PKG} / ${NSCQ_PKG} / nvlsm / infiniband-diags; skipping fabric manager." fi From 5bd81106f19996d36443af9f7afe3a9410328a3a Mon Sep 17 00:00:00 2001 From: Abhinav Nagaraj <2302780+abhinavnagaraj@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:52:32 -0700 Subject: [PATCH 32/34] feat(pam): bake libpam-pwquality into Ubuntu base image (#702) Adds libpam-pwquality to the Ubuntu apt install list so the pam_pwquality.so module is available for password-complexity enforcement in downstream user-data stages (e.g. AI Launchpad's default password policy in launchpad-ai#314). Airgap-safe: package is baked into the base image, not fetched at install. libpam-modules 1.4.0 on Ubuntu 24.04 does not support pam_exec + expose_authtok for password-type stanzas, so pam_pwquality is the correct primitive here. --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index afbb22a6..d9a5b860 100644 --- a/Earthfile +++ b/Earthfile @@ -786,7 +786,7 @@ base-image: END RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate -y + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate libpam-pwquality -y LET APT_UPGRADE_FLAGS="-y" IF [ "$UPDATE_KERNEL" = "false" ] From 62441716f09395164f113d1063dcce6223012d67 Mon Sep 17 00:00:00 2001 From: Nianyu Shen Date: Tue, 4 Aug 2026 12:33:41 -0700 Subject: [PATCH 33/34] AIL-376: feat(gpu): default AMD GPU driver to 31.40 (ROCm 7.14 GA) + amd-smi host CLI (#727) * feat(gpu): default AMD driver to 31.40 (ROCm 7.14 GA) + add amd-smi host CLI Bump the default AMDGPU_DRIVER_RELEASE from 7.2.1 to 31.40 across the Earthfile, install-amdgpu-drivers.sh, prebuild-amdgpu-artifact.sh, earthly.sh, and .arg.template. 31.40 is ROCm 7.14 GA (amdgpu-dkms 6.19.14) -- a production driver (7.14 went GA 2026-07-15), the baseline for GPU Operator v1.5.1 and for MI350P + Radeon AI PRO (RDNA4). The prior "31.x = tech preview" wording referred to the ROCm 7.13.0 preview and is now obsolete; docs corrected with an operator<->driver compatibility matrix. Fleets staying on Operator v1.5.0 can pin 7.2.1. Add amd-smi/rocm-smi host CLI for parity with nvidia-smi, gated by AMDGPU_INSTALL_SMI (default true). New install_amd_smi/ensure_rocm_repo helpers install amd-smi-lib (best-effort) and symlink amd-smi + rocm-smi into /usr/bin; recorded in the /etc/canvos/amdgpu-driver-source marker. Wired into all three script paths (inbox, dkms-artifact, dkms-in-buildkit) and both Earthfile invocations. The prebuild container sets AMDGPU_INSTALL_SMI=false since the artifact only carries kernel modules; the image-side artifact branch installs the CLI. Also fixes a stale AMDGPU_REBUILD_INITRD default (false, not true) in the docs. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(gpu): guard amd-smi install so best-effort SMI never aborts the build install_amd_smi was called bare under `set -eo pipefail`, so the symlink loop's `bin="$(ls /opt/rocm*/bin/${tool} ... | sort -V | tail -1)"` aborts the whole image build whenever a tool binary is absent -- notably the rocm-smi-lib fallback path the function itself creates. This defeats the function's documented "best-effort, never fails the build" contract. Guard the three call sites with `|| true`; a `||`-guarded function body disables `set -e` throughout, so the internal ls/ln/chmod steps can no longer kill the build. Verified: happy path still links both amd-smi and rocm-smi; the fallback links amd-smi and skips the absent rocm-smi. * fix(boot): drop pci=realloc=off so kernel PCI/BAR reallocation is not disabled pci=realloc=off pins the firmware-assigned PCI resource layout and disables kernel-side reallocation, which can prevent large / resizable-BAR windows from being allocated for GPUs. Remove it from the installer ISO grub cmdline (all three menu entries) and from the installed-OS bootargs sed. The nouveau + qat_4xxx blacklists are unaffected. * fix(gpu): install libdrm-amdgpu1 so amd-smi can enumerate GPUs amd-smi dlopens libdrm_amdgpu.so.1 at runtime to enumerate GPUs; it is not a linked dependency, so apt never pulled it (and --no-install-recommends would drop it regardless). On a booted node amd-smi failed with "Fail to open libdrm_amdgpu.so.1 ... Unable to detect any GPU devices" even with amdgpu loaded (rocm-smi, which reads sysfs directly, worked fine). Install libdrm-amdgpu1 (base Ubuntu repo, provides the SONAME) explicitly alongside amd-smi-lib in both the combined and fallback install paths. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(gpu): install amd-smi from repo.amd.com (amdrocm-amdsmi) matching the driver The legacy repo.radeon.com/rocm/apt tops out at ROCm 7.2.4 (amd-smi 26.2.2), older than the 31.40 driver (ROCm 7.14). AMD's current amd-smi install docs point at repo.amd.com/rocm/packages-multi-arch, which ships amdrocm-amdsmi 7.14.0 -- the user-space matching the driver. ensure_rocm_repo now prefers amdrocm-amdsmi from repo.amd.com (keyed by Ubuntu version, noble -> ubuntu2404) and falls back to amd-smi-lib from repo.radeon.com when unavailable (e.g. jammy). install_amd_smi installs the resolved package plus libdrm-amdgpu1 and symlinks the binary from either the /opt/rocm*/core-*/bin (core-SDK) or /opt/rocm*/bin (legacy) layout into /usr/bin. Also dearmor the ROCm key with `gpg --dearmor --yes` so the fallback can overwrite the keyring written by the repo.amd.com attempt -- otherwise gpg refuses to overwrite and the fallback silently fails to fetch its key. Verified in noble containers: primary installs amdrocm-amdsmi 7.14.0-3 (binary /opt/rocm/core-7.14/bin/amd-smi); forced fallback installs amd-smi-lib 26.2.2 (/opt/rocm/bin/amd-smi). Both land amd-smi on PATH with libdrm_amdgpu.so.1. * refactor(gpu): drop deprecated rocm-smi; ship amd-smi only amd-smi is the modern, actively-maintained superset that supersedes the legacy Python rocm-smi (AMD's recommended tool for Instinct). The install already pulls amdrocm-amdsmi from repo.amd.com (user-space matched to the driver, e.g. 7.14 for the 31.40 driver) with a legacy amd-smi-lib fallback; this drops rocm-smi from the symlink loop and updates the header, .arg.template, and docs accordingly. rocm-smi pulled Python deps and is being phased out, so removing it also keeps the image leaner. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Vipin Sharma Co-authored-by: Claude Opus 4.8 (1M context) --- .arg.template | 22 ++-- Earthfile | 25 ++-- docs/amd-gpu-airgapped.md | 95 +++++++------- earthly.sh | 2 +- overlay/files-iso/boot/grub2/grub.cfg | 6 +- scripts/install-amdgpu-drivers.sh | 172 +++++++++++++++++++++++--- scripts/prebuild-amdgpu-artifact.sh | 5 +- 7 files changed, 245 insertions(+), 82 deletions(-) diff --git a/.arg.template b/.arg.template index 82d49c6a..53f12e29 100644 --- a/.arg.template +++ b/.arg.template @@ -56,14 +56,20 @@ FORCE_INTERACTIVE_INSTALL=false # # and skips the AMD apt repo entirely — use # # when the DKMS build fails against your image # # kernel (see docs). -# AMDGPU_DRIVER_RELEASE=7.2.1 # Only used with dkms mode. amdgpu-install -# # release marker. Default 7.2.1 pairs with -# # GPU Operator v1.5.0 (ROCm 7.2.1). AMD also -# # publishes driver-release-marker paths like -# # 30.30.1 / 30.30.4 / 31.30; either form works. -# # 31.x is tech-preview -- do not mix with a -# # production operator. See the version- -# # alignment table in docs/amd-gpu-airgapped.md. +# AMDGPU_DRIVER_RELEASE=31.40 # Only used with dkms mode. amdgpu-install +# # release marker. Default 31.40 = ROCm 7.14 +# # GA (amdgpu 6.19.14), production driver for +# # GPU Operator v1.5.1 and MI350P / Radeon AI +# # PRO (RDNA4). For an older fleet on Operator +# # v1.5.0 use 7.2.1 (amdgpu 6.16.13). AMD +# # publishes both ROCm-alias (7.2.1, 7.2.4) and +# # marker (30.30.x, 31.40) paths; either works. +# # See the compatibility table in docs/. +# AMDGPU_INSTALL_SMI=true # Default true. Install the amd-smi host CLI on +# # PATH (parity with nvidia-smi), version-matched +# # to the driver (amdrocm-amdsmi from repo.amd.com). +# # rocm-smi is deprecated and NOT installed. +# # Best-effort; a small slice of ROCm user-space. # AMDGPU_REBUILD_INITRD=false # Default false. amdgpu is intentionally # # kept out of the initrd (multi-GPU init # # emits enough udev events to time out diff --git a/Earthfile b/Earthfile index d9a5b860..d8618413 100644 --- a/Earthfile +++ b/Earthfile @@ -84,12 +84,14 @@ ARG INSTALL_AMD_GPU_DRIVERS=false # fails against your image kernel and you accept the in-tree driver's feature set. ARG AMDGPU_DRIVER_SOURCE=dkms # amdgpu-install release marker (URL segment under repo.radeon.com/amdgpu-install//). -# Default 7.2.1 pairs with AMD GPU Operator v1.5.0 (per its release notes) and installs -# amdgpu-dkms 6.16.13 (30.30.1 line). Empirically builds cleanly against Linux kernels -# through 6.17 on Ubuntu 24.04. AMD publishes both ROCm-alias (7.2.1) and driver-release- -# marker (30.30.1, 31.30) URL segments; either form is accepted here. The 31.x line is -# tech-preview -- do not mix with a production operator. See docs/amd-gpu-airgapped.md. -ARG AMDGPU_DRIVER_RELEASE=7.2.1 +# Default 31.40 = ROCm 7.14 GA (amdgpu 6.19.14), the PRODUCTION driver for AMD GPU +# Operator v1.5.1 and the baseline for MI350P + Radeon AI PRO (RDNA4). (7.14 went GA +# on 2026-07-15; the earlier "31.x = tech-preview" note referred to the ROCm 7.13.0 +# preview and is obsolete.) For an older fleet staying on Operator v1.5.0, use 7.2.1 +# (amdgpu 6.16.13, 30.30.1 line). AMD publishes both ROCm-alias (7.2.1, 7.2.4) and +# driver-release-marker (30.30.x, 31.40) URL segments; either form is accepted here. +# See docs/amd-gpu-airgapped.md for the operator<->driver compatibility matrix. +ARG AMDGPU_DRIVER_RELEASE=31.40 # Path to a driver artifact produced by scripts/prebuild-amdgpu-artifact.sh # on the build host. Threaded in by earthly.sh when INSTALL_AMD_GPU_DRIVERS=true # and AMDGPU_DRIVER_SOURCE=dkms. When set, the base-image AMD block skips the @@ -102,6 +104,9 @@ ARG AMDGPU_ARTIFACT_PATH="" # the node into emergency mode). amdgpu loads after switch-root via # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. ARG AMDGPU_REBUILD_INITRD=false +# Install the amd-smi / rocm-smi host CLI on PATH (parity with nvidia-smi). Default +# true. Pulls a small slice of ROCm user-space from repo.radeon.com/rocm; best-effort. +ARG AMDGPU_INSTALL_SMI=true # NVIDIA and AMD driver pre-install are mutually exclusive within a single image. IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] && [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] @@ -884,6 +889,7 @@ base-image: AMDGPU_DRIVER_SOURCE=dkms \ AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_INSTALL_SMI="$AMDGPU_INSTALL_SMI" \ AMDGPU_ARTIFACT_PATH=/tmp/amdgpu-artifact.tar.gz \ /tmp/install-amdgpu-drivers.sh && \ rm -f /tmp/install-amdgpu-drivers.sh /tmp/amdgpu-artifact.tar.gz @@ -900,6 +906,7 @@ base-image: AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_INSTALL_SMI="$AMDGPU_INSTALL_SMI" \ /tmp/install-amdgpu-drivers.sh && \ rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh END @@ -983,12 +990,8 @@ base-image: # qat_4xxx: on Xeon Scalable 4th/5th gen hosts with QAT devices, # udev auto-loads qat_4xxx in initramfs and its probe/firmware-load # stalls boot for minutes. CanvOS does not consume QAT acceleration. - # - # pci=realloc=off: firmware-assigned PCI resource layout is - # authoritative; kernel-side reallocation has caused BAR conflicts - # on some server platforms. Mirrors the installer ISO cmdline. RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ - sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 pci=realloc=off"|' /etc/cos/bootargs.cfg; \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0"|' /etc/cos/bootargs.cfg; \ fi END diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md index 4a3c48ae..692a9ad2 100644 --- a/docs/amd-gpu-airgapped.md +++ b/docs/amd-gpu-airgapped.md @@ -21,7 +21,7 @@ The codename is derived from the image at build time. | Mode | What ships in the image | When to use | | --- | --- | --- | | `dkms` (default) | AMD's `amdgpu-dkms` source is DKMS-built against the image kernel and lands under `/lib/modules//updates/dkms/`. | Recommended for Instinct/MI silicon. The AMD out-of-tree driver typically carries newer SMU firmware interfaces and per-SKU support ahead of what the in-tree amdgpu has. | -| `inbox` | No AMD apt repo is added; the script only ensures the in-tree `amdgpu` module (shipped in `linux-modules-`) autoloads. | Fallback when the DKMS build fails against your image kernel — e.g. AMD hasn't yet published a driver release whose source builds against a very new kernel. Requires accepting the in-tree driver's feature set. | +| `inbox` | No AMD **driver** repo is added; the script only ensures the in-tree `amdgpu` module (shipped in `linux-modules-`) autoloads. (The ROCm repo is still added *only* for `amd-smi` if `AMDGPU_INSTALL_SMI=true`.) | Fallback when the DKMS build fails against your image kernel — e.g. AMD hasn't yet published a driver release whose source builds against a very new kernel. Requires accepting the in-tree driver's feature set. | Both modes still require `driver.enable=false` at the Helm layer — the operator does not build a driver either way. A marker at @@ -38,9 +38,11 @@ does not build a driver either way. A marker at | Component | Where it lives | Who installs it | | --- | --- | --- | | amdgpu kernel module (`amdgpu`) + firmware | **In the OS image** | **This script (build time)** | -| ROCm user-space, device-plugin, node-labeller, metrics-exporter | Container images | AMD GPU Operator (from your content bundle) | +| `amd-smi` host diagnostic CLI (`AMDGPU_INSTALL_SMI=true`) | **In the OS image** (`/usr/bin`) | **This script (build time)** | +| Full ROCm user-space, device-plugin, node-labeller, metrics-exporter | Container images | AMD GPU Operator (from your content bundle) | -The OS carries only the kernel driver; everything else is a container image you +The OS carries only the kernel driver (plus the small `amd-smi` CLI for +troubleshooting); everything else is a container image you mirror into your Palette content bundle. With `driver.enable=false` the operator "directly uses inbox or pre-installed AMD GPU drivers" and only deploys the device-plugin / node-labeller / metrics-exporter. @@ -118,7 +120,8 @@ autoloads. INSTALL_AMD_GPU_DRIVERS=true AMDGPU_DRIVER_SOURCE=dkms # or "inbox" — see modes above - AMDGPU_DRIVER_RELEASE=7.2.1 # pairs with GPU Operator v1.5.0 (dkms mode only) + AMDGPU_DRIVER_RELEASE=31.40 # ROCm 7.14 GA; pairs with GPU Operator v1.5.1 (dkms mode only) + AMDGPU_INSTALL_SMI=true # install amd-smi host CLI (parity with nvidia-smi; rocm-smi not installed) ``` 2. Build as usual, e.g.: @@ -132,7 +135,7 @@ autoloads. ```sh ./earthly.sh +base-image --ARCH=amd64 \ --INSTALL_AMD_GPU_DRIVERS=true \ - --AMDGPU_DRIVER_RELEASE=7.2.1 + --AMDGPU_DRIVER_RELEASE=31.40 ``` If the `dkms` build fails on your image kernel (see the mapping table @@ -150,13 +153,14 @@ autoloads. | --- | --- | --- | | `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Enables the AMD pre-install pipeline. | | `AMDGPU_DRIVER_SOURCE` | `dkms` | `dkms` (build AMD's out-of-tree driver against the image kernel) or `inbox` (skip the AMD repo and use the in-tree amdgpu). See modes above. | -| `AMDGPU_DRIVER_RELEASE` | `7.2.1` | **`dkms` mode only.** `amdgpu-install` URL segment under `repo.radeon.com/amdgpu-install//`. AMD publishes both ROCm-alias paths (e.g. `7.2.1`, `7.2.4`) and driver-release-marker paths (e.g. `30.30.1`, `30.30.4`, `31.30`) — either form works. Default `7.2.1` pairs with **GPU Operator v1.5.0** (per AMD's release notes) → **ROCm 7.2.1** → **amdgpu-dkms 6.16.13** (30.30.1 build). | -| `AMDGPU_REBUILD_INITRD` | `true` | Rebuild the initrd for the image kernel. | +| `AMDGPU_DRIVER_RELEASE` | `31.40` | **`dkms` mode only.** `amdgpu-install` URL segment under `repo.radeon.com/amdgpu-install//`. AMD publishes both ROCm-alias paths (e.g. `7.2.1`, `7.2.4`) and driver-release-marker paths (e.g. `30.30.x`, `31.40`) — either form works. Default `31.40` = **ROCm 7.14 GA** → **amdgpu-dkms 6.19.14**, the production driver for **GPU Operator v1.5.1** and the baseline for MI350P + Radeon AI PRO (RDNA4). For an older fleet on **v1.5.0**, use `7.2.1` (amdgpu-dkms 6.16.13). | +| `AMDGPU_INSTALL_SMI` | `true` | Install the `amd-smi` host CLI and symlink into `/usr/bin`, for parity with `nvidia-smi`. Prefers `amdrocm-amdsmi` from `repo.amd.com` (user-space matched to recent drivers, e.g. 7.14 for the `31.40` driver), falling back to `amd-smi-lib` from the legacy `repo.radeon.com/rocm/apt` (7.2.x). `rocm-smi` is deprecated and **not** installed. Best-effort — a failure warns but does not fail the build. | +| `AMDGPU_REBUILD_INITRD` | `false` | Rebuild the initrd for the image kernel. Default `false`: amdgpu is intentionally kept out of the initrd (multi-GPU init can time out dracut-initqueue); it loads after switch-root via `modules-load.d`. | -### Version alignment across the stack (snapshot, 2026-07-11) +### Version alignment across the stack -Five things have to line up to have a supportable node. Start from the operator -version you bundle and follow AMD's release notes / compat matrix from there: +Several things have to line up to have a supportable node. Start from the +operator version you bundle and follow AMD's release notes / compat matrix: ``` GPU Operator ─┐ AMD release notes pair the operator with a specific @@ -170,48 +174,50 @@ version you bundle and follow AMD's release notes / compat matrix from there: Kubernetes version — validated per operator release ``` -**Two parallel driver tracks** — do not mix them: +#### Operator ↔ driver compatibility -| Track | `AMDGPU_DRIVER_RELEASE` values | ROCm user-space | Paired GPU Operator | -| --- | --- | --- | --- | -| **Production** | `7.2.1` (= `30.30.1`), `7.2.4` (= `30.30.4`), etc. | ROCm 7.2.x | **v1.5.0 (what CanvOS bundles)** | -| Tech preview | `31.10` / `31.20` / `31.30` | ROCm 7.13.0 tech-preview | not yet paired with a released operator | +| GPU Operator | `AMDGPU_DRIVER_RELEASE` | amdgpu-dkms | ROCm | Notes | +| --- | --- | --- | --- | --- | +| **v1.5.1** (latest) | **`31.40`** (default) | `6.19.14` | **7.14 GA** | Production. Baseline for **MI350P** + **Radeon AI PRO (RDNA4)**. DRA driver needs `31.40`+. | +| v1.5.0 | `7.2.1` (= `30.30.1`) | `6.16.13` | 7.2.1 | Production. MI300X/MI325X, MI350X/MI355X. | +| v1.5.0 | `7.2.4` (= `30.30.4`) | `6.16.13` | 7.2.4 | Production patch of the 7.2.x line. | -Authoritative references: -- [AMD GPU Operator v1.5.0 release notes](https://instinct.docs.amd.com/projects/gpu-operator/en/main/releasenotes.html#gpu-operator-v1-5-0-release-notes) -- [ROCm user↔kernel compat matrix](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html) -- [ROCm on Linux system requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html) -- Repo index: +> **The old "31.x = tech preview" note is obsolete.** `31.x` referred to the +> **ROCm 7.13.0 preview**. **ROCm 7.14.0 went GA on 2026-07-15**, and marker +> `31.40` (amdgpu 6.19.14) is a **production** driver. MI350X/MI355X/MI350P +> (gfx950) and Radeon AI PRO R9700 (RDNA4/gfx1201) are all in the ROCm 7.14 +> production hardware matrix. The **only** remaining preview caveat in Operator +> v1.5.1 is **Auto Node Remediation (ANR)** — "not fully validated on MI350P +> and Radeon AI platforms in this beta release" — which is an optional operator +> *feature*, not the driver. -Snapshot of what `repo.radeon.com/amdgpu//ubuntu/dists/noble/…/Packages` -publishes for `amdgpu-dkms`: +**How to pick:** match the driver to the operator you bundle. +- Standardizing on **Operator v1.5.1** (incl. MI350P / RDNA4) → keep the default **`31.40`**. +- Staying on **Operator v1.5.0** with MI300X/MI325X → set **`AMDGPU_DRIVER_RELEASE=7.2.1`**. -| `AMDGPU_DRIVER_RELEASE` | `amdgpu-dkms` build | Track | Paired with | -| --- | --- | --- | --- | -| **`7.2.1` (= `30.30.1`, default)** | `6.16.13-2303411` | Production | ROCm 7.2.1 → GPU Operator v1.5.0 | -| `7.2.4` (= `30.30.4`) | `6.16.13-2341068` | Production | ROCm 7.2.4 | -| `31.10` | `6.18.4` | Tech preview | ROCm 7.13.0 tech-preview | -| `31.30` | `6.19.4` | Tech preview | ROCm 7.13.0 tech-preview | +Driver and operator move together — don't bump one without the other. -The whole 30.30.x line uses the same driver *source* (`6.16.13`); only the build -number and paired user-space differ. Empirically, this source **builds cleanly -against Ubuntu 24.04's edge kernel 6.17** — the "EFI variables are not supported -on this system" line printed during postinst is a cosmetic mokutil warning -(sign_tool step); DKMS proceeds and lands the module under `updates/dkms/`. +Authoritative references: +- [AMD GPU Operator release notes](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/release-notes.html) +- [ROCm compatibility matrix](https://rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html) +- [ROCm user↔kernel compat matrix](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html) +- Repo index: -**How to pick:** default to the release marker paired with the operator you're -bundling. Bump only when you're also moving the operator to a paired version. -Do not switch to `31.x` for kernel-newness alone — that crosses into tech -preview and won't be validated with a production operator. +The 30.30.x line shares one driver *source* (`6.16.13`) and builds cleanly +against Ubuntu 24.04's 6.17 kernel; the `31.40` source (`6.19.14`) is the ROCm +7.14 driver. The "EFI variables are not supported on this system" line during +postinst is a cosmetic mokutil warning (sign_tool step); DKMS proceeds and lands +the module under `updates/dkms/`. ### When the DKMS build fails Common causes: 1. **Kernel outside the driver's supported window** — `make.log` shows - `configure: cannot detect CFLAGS` or unresolved kernel symbols. Prefer - moving the image kernel into range (or the operator/ROCm/driver combo up - as a set) over jumping to a tech-preview driver. + `configure: cannot detect CFLAGS` or unresolved kernel symbols. Move the + image kernel into range, or move the operator/ROCm/driver combo up as a + paired set — don't change `AMDGPU_DRIVER_RELEASE` in isolation just to chase + a newer kernel. 2. **`linux-headers-` not installed for the image kernel** — check the earlier log lines from `install-kernel-headers.sh`. Fix the headers. 3. **DKMS module signing (mokutil) failure in the container** — surfaces as @@ -237,11 +243,14 @@ newer silicon). lsmod | grep amdgpu dmesg | grep -i amdgpu ls /sys/class/kfd 2>/dev/null && echo "KFD present" -cat /etc/canvos/amdgpu-driver-source # which mode ran + release info +cat /etc/canvos/amdgpu-driver-source # which mode ran + release + AMDGPU_SMI=yes/no # In dkms mode, expect a module under /lib/modules//updates/dkms/ find /lib/modules/$(uname -r)/updates -name 'amdgpu.ko*' 2>/dev/null -# If you also bundle ROCm user-space tooling: -# rocminfo ; amd-smi list +# amd-smi is baked into the OS image when AMDGPU_INSTALL_SMI=true (default): +amd-smi version # user-space version (should match the driver, e.g. 7.14 for 31.40) +amd-smi list # like `nvidia-smi -L` — enumerates GPUs from the host +amd-smi monitor # live utilization +# (rocm-smi is deprecated and intentionally not installed — amd-smi supersedes it) ``` --- diff --git a/earthly.sh b/earthly.sh index d18ff737..1147348e 100755 --- a/earthly.sh +++ b/earthly.sh @@ -367,7 +367,7 @@ if [ "$INSTALL_AMD_GPU_DRIVERS_EFFECTIVE" = "true" ] && [ "$AMDGPU_DRIVER_SOURCE fi AMDGPU_DRIVER_RELEASE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_RELEASE "$@")" - AMDGPU_DRIVER_RELEASE_EFFECTIVE="${AMDGPU_DRIVER_RELEASE_EFFECTIVE:-${AMDGPU_DRIVER_RELEASE:-7.2.1}}" + AMDGPU_DRIVER_RELEASE_EFFECTIVE="${AMDGPU_DRIVER_RELEASE_EFFECTIVE:-${AMDGPU_DRIVER_RELEASE:-31.40}}" echo "=== Pre-building AMD amdgpu driver (dkms mode) ===" echo " BASE_IMAGE: $AMDGPU_BASE_IMAGE" diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index a3bb3162..1aaa44fc 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -13,21 +13,21 @@ if [ -f ${font} ];then fi menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install echo Loading initrd... $initrd ($root)/boot/initrd } diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh index 6d2889e8..994ae974 100755 --- a/scripts/install-amdgpu-drivers.sh +++ b/scripts/install-amdgpu-drivers.sh @@ -32,6 +32,8 @@ # * linux-modules-extra for the image kernel # * amdgpu-dkms + amdgpu-dkms-firmware, built against the IMAGE kernel # * amdgpu module autoload + initrd refresh +# * amd-smi host CLI on PATH (parity with nvidia-smi) when +# AMDGPU_INSTALL_SMI=true (default). rocm-smi is deprecated and NOT installed. # * a marker at /etc/canvos/amdgpu-driver-source recording which mode ran # # WHAT THIS DOES *NOT* COVER (both modes) -- ship these as container images in @@ -53,13 +55,26 @@ # AMDGPU_DRIVER_RELEASE amdgpu-install release marker (URL segment under # repo.radeon.com/amdgpu-install//). AMD publishes # both ROCm-alias paths (7.2.1, 7.2.4) and driver- -# release-marker paths (30.30.1, 30.30.4, 31.30); -# either form is accepted. Default: 7.2.1 -- pairs -# with GPU Operator v1.5.0 per its release notes. -# The 31.x line is tech-preview and pairs only with -# ROCm 7.13.0 tech-preview; do not mix with a -# production operator. Ignored in inbox mode. See -# docs/amd-gpu-airgapped.md for the compat matrix. +# release-marker paths (30.30.x, 31.40); either form +# is accepted. Default: 31.40 -- ROCm 7.14 GA (amdgpu +# 6.19.14), a PRODUCTION driver (7.14 went GA on +# 2026-07-15; the earlier "31.x = tech-preview" note +# referred to the ROCm 7.13.0 preview and is obsolete). +# 31.40 is the baseline for GPU Operator v1.5.1 and +# for MI350P + Radeon AI PRO (RDNA4). For an older +# fleet staying on Operator v1.5.0, use 7.2.1 (amdgpu +# 6.16.13, 30.30.1 line). Ignored in inbox mode. See +# docs/amd-gpu-airgapped.md for the operator<->driver +# compatibility matrix. +# AMDGPU_INSTALL_SMI true | false. Default: true. Install the amd-smi +# host CLI and symlink into /usr/bin, for parity with +# nvidia-smi. Prefers amdrocm-amdsmi from repo.amd.com +# (user-space matching recent drivers, e.g. 7.14 for +# the 31.40 driver), falling back to amd-smi-lib from +# repo.radeon.com. Best-effort: a failure warns but +# does not fail the build. +# AMDGPU_ROCM_APT_VERSION Legacy repo.radeon.com/rocm/apt version, used only +# for the amd-smi-lib fallback. Default: latest. # AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. # Default: false. amdgpu is intentionally NOT # included in the initrd (see the dracut omit @@ -75,12 +90,121 @@ log() { echo "[install-amdgpu-drivers] $*"; } warn() { echo "[install-amdgpu-drivers] WARNING: $*" >&2; } die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } +# --------------------------------------------------------------------------- +# ensure_rocm_repo: make an amd-smi package installable and record its name in +# AMDSMI_PKG. Preference order: +# 1. already resolvable (e.g. the amdgpu-install deb configured a repo in the +# in-buildkit dkms path) -- prefer amdrocm-amdsmi, else amd-smi-lib; +# 2. the current ROCm repo repo.amd.com/rocm/packages-multi-arch, which ships +# amdrocm-amdsmi matching recent drivers (7.14 user-space for the 31.40 +# driver) -- this is the repo AMD's amd-smi install docs point to, keyed by +# Ubuntu version (noble -> ubuntu2404); +# 3. the legacy repo.radeon.com/rocm/apt, which ships amd-smi-lib (7.2.x). +# Returns 0 with AMDSMI_PKG set on success, non-zero if no repo yields a package. +# --------------------------------------------------------------------------- +ensure_rocm_repo() { + apt-get install -y --no-install-recommends ca-certificates curl gnupg >/dev/null 2>&1 || true + apt-get update >/dev/null 2>&1 || true + + # (1) Already resolvable (repo configured by amdgpu-install). Prefer the + # modern amdrocm-amdsmi; fall back to amd-smi-lib. + if apt-cache show amdrocm-amdsmi >/dev/null 2>&1; then AMDSMI_PKG="amdrocm-amdsmi"; return 0; fi + if apt-cache show amd-smi-lib >/dev/null 2>&1; then AMDSMI_PKG="amd-smi-lib"; return 0; fi + + install -d -m 0755 /etc/apt/keyrings + + # (2) Preferred: repo.amd.com (current ROCm packaging), keyed by Ubuntu + # version. Ships amd-smi user-space matching recent drivers. + if [ -n "${ubuntu_ver:-}" ] \ + && curl -fsSL https://repo.amd.com/rocm/packages-multi-arch/gpg/rocm.gpg 2>/dev/null \ + | gpg --dearmor --yes -o /etc/apt/keyrings/rocm.gpg 2>/dev/null; then + chmod a+r /etc/apt/keyrings/rocm.gpg + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.amd.com/rocm/packages-multi-arch/ubuntu${ubuntu_ver} stable main" \ + > /etc/apt/sources.list.d/rocm.list + apt-get update >/dev/null 2>&1 || true + if apt-cache show amdrocm-amdsmi >/dev/null 2>&1; then + log "Using amdrocm-amdsmi from repo.amd.com (ubuntu${ubuntu_ver})." + AMDSMI_PKG="amdrocm-amdsmi" + return 0 + fi + warn "amdrocm-amdsmi not found in repo.amd.com/ubuntu${ubuntu_ver}; trying legacy ROCm apt repo." + else + warn "could not add repo.amd.com ROCm repo; trying legacy ROCm apt repo." + fi + + # (3) Fallback: legacy repo.radeon.com/rocm/apt (ships amd-smi-lib). + log "Adding legacy ROCm apt repo (repo.radeon.com/rocm/apt/${AMDGPU_ROCM_APT_VERSION}) for amd-smi ..." + if ! curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key 2>/dev/null \ + | gpg --dearmor --yes -o /etc/apt/keyrings/rocm.gpg 2>/dev/null; then + warn "could not fetch/dearmor the legacy ROCm gpg key." + return 1 + fi + chmod a+r /etc/apt/keyrings/rocm.gpg + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${AMDGPU_ROCM_APT_VERSION} ${codename} main" \ + > /etc/apt/sources.list.d/rocm.list + apt-get update >/dev/null 2>&1 || { warn "apt-get update after adding legacy ROCm repo failed."; return 1; } + apt-cache show amd-smi-lib >/dev/null 2>&1 || { warn "amd-smi-lib not found even after adding the legacy ROCm repo."; return 1; } + AMDSMI_PKG="amd-smi-lib" + return 0 +} + +# --------------------------------------------------------------------------- +# install_amd_smi: install the amd-smi host CLI (amdrocm-amdsmi from repo.amd.com +# where available, else legacy amd-smi-lib) and put it on PATH at /usr/bin, +# mirroring how nvidia-utils gives /usr/bin/nvidia-smi. Best-effort by design -- +# a diagnostic tool must never fail the image build. +# --------------------------------------------------------------------------- +install_amd_smi() { + if [ "${AMDGPU_INSTALL_SMI}" != "true" ]; then + log "AMDGPU_INSTALL_SMI=false; skipping amd-smi host CLI." + return 0 + fi + log "Installing amd-smi host CLI (parity with nvidia-smi) ..." + AMDSMI_PKG="" + if ! ensure_rocm_repo || [ -z "${AMDSMI_PKG}" ]; then + warn "ROCm repo unavailable; skipping amd-smi. Set AMDGPU_INSTALL_SMI=false to silence." + return 0 + fi + # amd-smi dlopens libdrm_amdgpu.so.1 at runtime to enumerate GPUs (it is NOT + # a linked dependency, so apt won't pull it -- and --no-install-recommends + # would drop it anyway). Without it amd-smi fails with + # "Fail to open libdrm_amdgpu.so.1 ... Unable to detect any GPU devices" + # even when amdgpu is loaded. libdrm-amdgpu1 (from the base Ubuntu repo) + # provides the SONAME, so install it explicitly alongside the SMI package. + log "Installing ${AMDSMI_PKG} + libdrm-amdgpu1 ..." + if ! apt-get install -y --no-install-recommends "${AMDSMI_PKG}" libdrm-amdgpu1; then + warn "${AMDSMI_PKG} + libdrm-amdgpu1 install failed; retrying with ${AMDSMI_PKG} only." + apt-get install -y --no-install-recommends "${AMDSMI_PKG}" \ + || { warn "${AMDSMI_PKG} install failed; continuing without amd-smi."; return 0; } + fi + # The amd-smi binary lands under /opt/rocm*/bin (legacy amd-smi-lib) or + # /opt/rocm*/core-*/bin (repo.amd.com core-SDK layout); neither is on the + # default PATH. Symlink it into /usr/bin so `amd-smi` works ootb like + # `nvidia-smi`. rocm-smi is intentionally NOT installed -- it is deprecated + # and fully superseded by amd-smi. + smi_bin="$(ls /opt/rocm*/core-*/bin/amd-smi /opt/rocm*/bin/amd-smi 2>/dev/null | sort -V | tail -1 || true)" + if [ -n "${smi_bin}" ]; then + ln -sf "${smi_bin}" /usr/bin/amd-smi + log "Linked ${smi_bin} -> /usr/bin/amd-smi" + fi + if [ -x /usr/bin/amd-smi ]; then + AMDGPU_SMI_INSTALLED="yes" + else + warn "amd-smi binary not found under /opt/rocm* after installing ${AMDSMI_PKG}." + fi +} + # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" -AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.40}" AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-false}" +AMDGPU_INSTALL_SMI="${AMDGPU_INSTALL_SMI:-true}" +AMDGPU_ROCM_APT_VERSION="${AMDGPU_ROCM_APT_VERSION:-latest}" +AMDGPU_SMI_INSTALLED="" # set to "yes" by install_amd_smi on success +AMDSMI_PKG="" # set by ensure_rocm_repo to the resolvable amd-smi package +ubuntu_ver="" # e.g. 2404 for repo.amd.com/.../ubuntu2404 (set from os-release below) case "${AMDGPU_DRIVER_SOURCE}" in dkms|inbox) ;; @@ -108,6 +232,8 @@ if [ -r /etc/os-release ]; then . /etc/os-release codename="${VERSION_CODENAME:-}" osid="${ID:-ubuntu}" + # repo.amd.com component tag: VERSION_ID 24.04 -> ubuntu2404, 22.04 -> ubuntu2204. + ubuntu_ver="$(printf '%s' "${VERSION_ID:-}" | tr -d '.')" fi [ -n "${codename}" ] || die "could not determine Ubuntu codename from /etc/os-release." @@ -168,7 +294,13 @@ EOF fi fi - printf 'AMDGPU_DRIVER_SOURCE=inbox\nKVER=%s\n' "${KVER}" > /etc/canvos/amdgpu-driver-source + install_amd_smi || true # best-effort; must never fail the build + + printf 'AMDGPU_DRIVER_SOURCE=inbox\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + apt-get clean || true + rm -rf /var/lib/apt/lists/* log "Done. Using in-tree amdgpu driver for kernel ${KVER}." log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." @@ -238,10 +370,15 @@ EOF fi fi + install_amd_smi || true # best-effort; must never fail the build + # Marker written by the prebuild is preserved from the tar. Overwrite # any prebuild-mode marker with the final in-image reality. - printf 'AMDGPU_DRIVER_SOURCE=dkms (artifact)\nAMDGPU_DRIVER_RELEASE=%s\nKVER=%s\n' \ - "${AMDGPU_DRIVER_RELEASE}" "${KVER}" > /etc/canvos/amdgpu-driver-source + printf 'AMDGPU_DRIVER_SOURCE=dkms (artifact)\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + apt-get clean || true + rm -rf /var/lib/apt/lists/* log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}, artifact) baked in for kernel ${KVER}." log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." @@ -517,10 +654,17 @@ elif [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/de fi # --------------------------------------------------------------------------- -# 11. Record what we did so ops can query it on-node. +# 11b. Install the amd-smi host CLI (parity with nvidia-smi). The amdgpu-install +# deb above may already have configured the ROCm repo; if not, install_amd_smi +# adds repo.amd.com. Best-effort -- never fails the build. +# --------------------------------------------------------------------------- +install_amd_smi || true # best-effort; must never fail the build + +# --------------------------------------------------------------------------- +# 11c. Record what we did so ops can query it on-node. # --------------------------------------------------------------------------- -printf 'AMDGPU_DRIVER_SOURCE=dkms\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_DKMS_MODULE=%s/%s\nKVER=%s\n' \ - "${AMDGPU_DRIVER_RELEASE}" "${mod}" "${ver}" "${KVER}" \ +printf 'AMDGPU_DRIVER_SOURCE=dkms\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_DKMS_MODULE=%s/%s\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${mod}" "${ver}" "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" \ > /etc/canvos/amdgpu-driver-source # --------------------------------------------------------------------------- diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh index e341d173..3c74deb6 100755 --- a/scripts/prebuild-amdgpu-artifact.sh +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -35,7 +35,7 @@ # # INPUTS (env vars; defaults mirror the Earthfile / .arg.template) # BASE_IMAGE kairos base image ref (REQUIRED) -# AMDGPU_DRIVER_RELEASE default: 7.2.1 (pairs with GPU Operator v1.5.0) +# AMDGPU_DRIVER_RELEASE default: 31.40 (ROCm 7.14 GA; pairs with GPU Operator v1.5.1) # AMDGPU_ARTIFACT_DIR default: ./build # AMDGPU_FORCE_REBUILD set to 1 to bypass cache # @@ -51,7 +51,7 @@ die() { echo "[prebuild-amdgpu] ERROR: $*" >&2; exit 1; } # --- inputs --------------------------------------------------------------- : "${BASE_IMAGE:?BASE_IMAGE must be set (kairos base image ref, e.g. us-docker.pkg.dev/palette-images/edge/kairos-ubuntu:24.04-core-amd64-generic-v4.0.4)}" -AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.40}" AMDGPU_ARTIFACT_DIR="${AMDGPU_ARTIFACT_DIR:-./build}" AMDGPU_FORCE_REBUILD="${AMDGPU_FORCE_REBUILD:-0}" @@ -104,6 +104,7 @@ docker run --rm --privileged \ -e AMDGPU_DRIVER_SOURCE=dkms \ -e AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE}" \ -e AMDGPU_REBUILD_INITRD=false \ + -e AMDGPU_INSTALL_SMI=false \ -e KVER_EXPECTED="${KVER}" \ -v "${INSTALL_SCRIPT}:/tmp/install-amdgpu-drivers.sh:ro" \ -v "${HEADERS_SCRIPT}:/tmp/install-kernel-headers.sh:ro" \ From 30d21418a4889c5d67010e923803e4d1506bf584 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Thu, 6 Aug 2026 15:56:22 -0700 Subject: [PATCH 34/34] =?UTF-8?q?AIL-418:=20feat(iso):=20switch=20osbuilde?= =?UTF-8?q?r=20=E2=86=92=20AuroraBoot=20v0.26.2=20for=20+build-iso=20and?= =?UTF-8?q?=20+build-uki-iso=20(#738)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update grub.cfg * feat(iso): switch +build-iso and +build-uki-iso to AuroraBoot kairos-io/osbuilder and kairos-io/enki are both archived, so the osbuilder-tools image we've built ISOs against for years has no maintained upstream. Switch to quay.io/kairos/auroraboot:v0.26.1 (kairos-io/AuroraBoot), the maintained successor. Only +build-iso and +build-uki-iso move. +uki-genkey and +build-provider-trustedboot-image still call the osbuilder entrypoint; they touch container/UKI artifacts, not ISOs, and can migrate separately once the ISO switch is boot-verified. Mapping (osbuilder -> AuroraBoot): /entrypoint.sh build-iso -> auroraboot build-iso --name -> --override-name --arch x86_64 -> --arch amd64 (Go arch names) enki build-uki -t iso -> auroraboot build-uki -t iso -k /keys (bundled) -> --public-keys / --sb-key / --sb-cert / --tpm-pcr-private-key (all still under /keys because the three secure-boot/* source dirs are flattened into the same COPY target) Bonuses from the AuroraBoot base image (Fedora 44): - xorriso 1.5.8.pl02 (osbuilder-tools ships 1.4.6, which the Supermicro ATEN BMC virtual CD refuses to boot). This is the xorriso fix vipsharm-isofix built from source; no longer needed. - Actively maintained; kairos-sdk fixes land here. Caveat: AuroraBoot still selects grubx64.efi.signed (installed-system variant, prefix /EFI/ubuntu) via kairos-sdk utils.GetEfiGrubFiles. That binary hangs immediately on Supermicro UEFI. This switch does not fix defect 1 from the UEFI investigation on its own. Verifying that on real hardware is the next step; if it reproduces, the CD-variant swap workaround from vipsharm-isofix 22e00cb / 0a80d30 gets ported. The grub.cfg cleanup (previous commit, ex-3faff86) is independent of the toolchain switch and rides along. Co-Authored-By: Claude Opus 4.7 * fix(iso): normalize AuroraBoot's ISO filename to \$ISO_NAME.iso AuroraBoot v0.26.1's --override-name is not honored for "dir:" sources (only image sources). A build-iso against dir:/build/image landed as kairos-ubuntu-24.04-core-amd64-generic-v4.0.4.iso in /iso/, and the subsequent RUN sha256sum \$ISO_NAME.iso > \$ISO_NAME.iso.sha256 failed with "No such file or directory". Rename /iso/*.iso to \$ISO_NAME.iso before checksumming, matching what +build-uki-iso has done all along. This decouples us from whatever naming scheme AuroraBoot uses internally. Co-Authored-By: Claude Opus 4.7 * fix(iso): locate AuroraBoot's ISO wherever it lands, not just /iso/ AuroraBoot v0.26.1's --output flag on build-iso is ignored for "dir:" sources: the log line "Generating iso '...' to '/tmp/auroraboot'" confirmed the ISO drops under /tmp/auroraboot, not /iso/, so `mv /iso/*.iso ...` failed with "cannot stat '/iso/*.iso'". Search /iso, /tmp/auroraboot, and (fall back) the whole filesystem for a .iso and move it into /iso/\$ISO_NAME.iso. Applied to both +build-iso and +build-uki-iso so we're covered if build-uki -d has the same bug. Co-Authored-By: Claude Opus 4.7 * chore(iso): drop dead --output/--override-name/-d flags and clean up rename Both --output on build-iso and -d on build-uki are silently ignored by AuroraBoot v0.26.1 for "dir:" sources -- the ISO always drops at /tmp/auroraboot/*.iso. Two builds in a row confirmed the location, so drop the fallback find-tree and just mv from /tmp/auroraboot directly. Also drops --override-name (also inert for dir: sources on build-iso) so the invocation reads honestly. No behavior change -- the previous fallback path already worked. This is just less noisy shell. Co-Authored-By: Claude Opus 4.7 * fix(iso): ship the CD-variant GRUB so ISOs boot in pure UEFI Ubuntu's grub-efi-*-signed package contains two signed GRUB builds: grubx64.efi.signed prefix /EFI/ubuntu - for systems installed to disk gcdx64.efi.signed prefix /boot/grub - for CD / removable media osbuilder's copyGrub walks a hardcoded candidate list (kairos-sdk utils.GetEfiGrubFiles), takes the first path that exists, and names the destination after the source. On Ubuntu that always resolves to grubx64.efi.signed - there is no gcd candidate in the list at all - so our ISOs ship the installed-system binary. That binary hangs immediately when launched from removable media on UEFI firmware. Reproduced on Supermicro with an ATEN BMC: the firmware mounts the ESP correctly and reads both binaries, then executes nothing. Launching either EFI\BOOT\bootx64.efi or EFI\BOOT\grubx64.efi by hand from the UEFI Shell hangs with no output, with Secure Boot disabled. Only Dual/CSM boot ever worked, because that path uses the BIOS El Torito image and never touches this binary. It is not a version problem: ubuntu-24.04.3-live-server ships the identical GRUB package version (2.12-1ubuntu7.3) and boots on the same hardware. Substituting its gcd build into our ESP makes GRUB run on every boot. Changes: - Earthfile (+iso-image): stage the CD variant over the path osbuilder reads. Applied to the ISO rootfs only - cloud/raw disk images are installed systems and must keep the original binary. Handles x86_64 and arm64, and is a no-op on bases that do not ship the gcd build. - overlay/files-iso/boot/grub/grub.cfg: new stub. The CD-variant GRUB resolves its /boot/grub prefix against the ISO9660 volume, so it needs a config there; without it GRUB starts and drops to a rescue prompt. Hands off to the existing /boot/grub2/grub.cfg. This is a workaround, not a fix: kairos-io/osbuilder and kairos-io/enki are both archived, so the candidate list cannot be corrected upstream. The durable option is migrating ISO builds to AuroraBoot, which is maintained but inherits the same defect via kairos-sdk. Verified by hand-patching an ISO (CD-variant GRUB + this stub + the grub.cfg fixes in the previous commit): boots unattended in pure UEFI, menu renders on the EFI console, kernel boots with visible output. NOT yet verified through a CanvOS pipeline build. * fix(iso): skip CD-variant GRUB swap for MAAS builds +kairos-raw-image invokes +iso-image with no arguments (Earthfile:1259), so IS_CLOUD_IMAGE stays at its default of false. The 22e00cb guard IS_CLOUD_IMAGE=false therefore fires on MAAS builds too, replacing grubx64.efi.signed with the CD variant. AuroraBoot then writes that into the raw disk image's ESP, and a MAAS-deployed machine boots into the CD-variant GRUB, which looks for /boot/grub/grub.cfg on the installed disk and drops to a rescue prompt when it isn't there. Widen the guard to also skip when IS_MAAS=true. IS_MAAS is a top-level ARG (Earthfile:124) that already propagates into +iso-image on MAAS builds, so no call-site change is needed. Co-Authored-By: Claude Opus 4.7 * fix(iso): post-process AuroraBoot ISO with xorriso to add GPT + inject overlay files AuroraBoot v0.26.1's build-iso has two ISO-side defects that block boot on strict UEFI firmwares: 1. Produces a plain-MBR ISO with El Torito + a 0xEF appended-partition ESP entry. Works on lenient firmwares, but VMware ESXi/Workstation, OVMF/QEMU, and some SuperMicro BMC virtual CD implementations reject it as "No compatible bootloader found" because they only look for the ESP via a GPT partition entry. Ubuntu 24.04.3's live ISO uses hybrid GPT + protective MBR for exactly this reason and boots everywhere. 2. --overlay-iso is silently a no-op for "dir:" sources -- files placed under /overlay do NOT make it onto the ISO tree. Diagnosed by comparing overlay/files-iso/boot/grub2/grub.cfg (Palette-branded) with the same path on the built ISO (Kairos-branded default), and by finding /boot/grub/grub.cfg entirely missing. That in turn drops the CD-variant GRUB stub (from 22e00cb), the customized menu, user-data, content bundles, cluster config, and edge_custom_config. Fix: xorriso repack after the auroraboot run. -boot_image any appended_part_as=gpt Converts the appended ESP into a GPT partition entry and emits a protective MBR. UEFI firmware can now discover it. -add $OVERLAY_SPECS -- Enumerates every file under /overlay and inject-adds each to the ISO tree at its expected location. Also functions as the "pending change" that lets xorriso commit (without a tree change it would refuse with "No image modifications pending"). -overwrite on Some overlay files (like /boot/grub2/grub.cfg) already exist on the ISO tree with AuroraBoot's default content; -overwrite on lets -add replace them cleanly. Verified on the jump host: repacked ISO boots to GRUB menu in QEMU + OVMF and in VMware VM (Ubuntu 24.04.3 also broken on that VM until GPT was added). Only touches +build-iso. +build-uki-iso uses auroraboot build-uki -t iso which is a different upstream code path and may or may not have the same defects -- to be evaluated when UKI ISOs are next boot-tested. Also filed for upstream: kairos-io/AuroraBoot issue on --overlay-iso being ignored, and kairos-io/kairos-sdk issue on GetEfiGrubFiles never picking the gcd/CD-variant signed binary. Co-Authored-By: Claude Opus 4.7 * revert: drop UEFI-ISO workarounds now that AuroraBoot v0.26.2 fixes them upstream kairos-io/AuroraBoot v0.26.2 (PR #713) fixes the two defects that were blocking UEFI-only ISO boot, exactly as our RCA identified: * appended_part_as=gpt is now passed to xorriso, so ISOs get a GPT header alongside the protective MBR (fixes VMware ESXi/Workstation EFI, OVMF/QEMU, some SuperMicro BMC virtual-CD firmwares that previously reported "No compatible bootloader found"). * Live ISOs now prefer gcdx64.efi.signed / gcdaa64.efi.signed (CD-variant grub with prefix /boot/grub and iso9660 baked in) over grubx64.efi.signed (installed-system grub, prefix /EFI/ubuntu with no iso9660). cleanupGrubName renames gcd*.efi to grub*.efi on copy so shim chainloading still works under Secure Boot. * A /boot/grub/grub.cfg stub is written by AuroraBoot itself, both in the EFI image (USB path) and on the ISO9660 root (CD path). * Bumps kairos-sdk to v0.25.2 which carries the sdk-side changes. With upstream fixed we can drop every local workaround: Earthfile (+iso-image): remove the CD-variant GRUB swap block (was: cp gcdx64.efi.signed over grubx64.efi.signed, guarded by IS_CLOUD_IMAGE=false && IS_MAAS=false). AuroraBoot now picks the right binary itself for live media, and doesn't touch the installed-system variant that raw-disk/MAAS builds depend on -- the MAAS guard becomes moot along with the swap. Earthfile (+build-iso): remove the xorriso post-process step that added GPT via appended_part_as=gpt and injected every overlay/ file with -add. GPT is now emitted natively; --overlay-iso keeps the same behaviour it had (documented separately; not blocking us because AuroraBoot's own /boot/grub/grub.cfg + /boot/grub2/ grub.cfg cover the boot chain). overlay/files-iso/boot/grub/grub.cfg: delete. Was our stub telling the CD-variant GRUB where to find its real config; AuroraBoot now writes an equivalent file itself. AURORABOOT_VERSION v0.26.1 -> v0.26.2. Verified path is now: FROM auroraboot:v0.26.2 -> build-iso dir:... -> mv /tmp/auroraboot/*.iso /iso/\$ISO_NAME.iso -> sha256sum. No post-processing. Not yet verified end-to-end (rebuild + boot in QEMU/OVMF and VMware UEFI expected) -- filing this as the trust-upstream state and will test right after. Co-Authored-By: Claude Opus 4.7 * fix(iso): xorriso-inject overlay into ISO because AuroraBoot's build-iso subcommand skips StepInjectCC Empirically: AuroraBoot v0.26.2 subcommand \`build-iso dir:...\` with --overlay-iso produces an ISO whose /boot/grub2/grub.cfg is AuroraBoot's default "Kairos"-branded template rather than our overlay's "Palette eXtended Kubernetes Edge Installer" version. Confirmed by extracting the file from the built ISO. Root cause (pkg/ops/iso.go, internal/cmd/build-iso.go): the build-iso subcommand runs only PrepDirs -> StepCopyCloudConfig -> StepDumpSource -> StepGenISO and never calls StepInjectCC. StepInjectCC is where AuroraBoot's InjectISO() function xorriso-maps the overlay directory onto the finalised ISO tree. Without it, --overlay-iso silently discards every file we put under /overlay/ -- not just the grub menu, but also user-data, cluster config (spc.tgz), content bundles, and edge_custom_config. Two ways to fix: 1. Switch +build-iso to AuroraBoot pipeline mode (docker run auroraboot --set container_image=... --set iso.overlay_iso=... --cloud-config ...). This invokes StepInjectCC. But it's a ~100-line rewrite involving DinD, container_image loading, and matches the pattern +cloud-image / +kairos-raw-image already use. 2. Do StepInjectCC's actual work ourselves inline. Its whole body is one xorriso command: xorriso -indev X -outdev X -map / \ -boot_image any replay No DinD, no rewrite. Same result. Going with (2). Small, isolated, uses the same xorriso invocation the upstream code path does, and can be trivially deleted the day StepInjectCC is wired into the subcommand path. Doesn't touch +build-uki-iso; that uses build-uki -t iso which is a separate code path. Filed separately as an upstream issue on kairos-io/AuroraBoot (subcommand build-iso should call StepInjectCC too, or --overlay-iso should be honored end-to-end within StepGenISO). Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Arvind Co-authored-by: Claude Opus 4.7 --- Earthfile | 97 ++++++++++++++++++++++----- overlay/files-iso/boot/grub2/grub.cfg | 39 ++++++++--- 2 files changed, 110 insertions(+), 26 deletions(-) diff --git a/Earthfile b/Earthfile index d8618413..4bc5e910 100644 --- a/Earthfile +++ b/Earthfile @@ -27,7 +27,7 @@ ARG RKE2_FLAVOR_TAG=rke2r1 ARG BASE_IMAGE_URL=quay.io/kairos ARG OSBUILDER_VERSION=v0.400.3 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION -ARG AURORABOOT_VERSION=v0.16.0 +ARG AURORABOOT_VERSION=v0.26.2 ARG AURORABOOT_IMAGE=quay.io/kairos/auroraboot:$AURORABOOT_VERSION ARG K3S_PROVIDER_VERSION=v4.9.1 ARG KUBEADM_PROVIDER_VERSION=v4.9.3 @@ -333,7 +333,11 @@ install-k8s: SAVE ARTIFACT --keep-ts /output/ . build-uki-iso: - FROM --platform=linux/${ARCH} $OSBUILDER_IMAGE + # Switched from quay.io/kairos/osbuilder-tools (archived kairos-io/osbuilder + # + kairos-io/enki) to AuroraBoot, which is the maintained successor. The + # build-iso and build-uki subcommands accept a "dir:" source, so the rootfs + # preparation path above is unchanged; only the final CLI invocation differs. + FROM --platform=linux/${ARCH} $AURORABOOT_IMAGE ENV ISO_NAME=${ISO_NAME} COPY overlay/files-iso/ /overlay/ COPY --if-exists +validate-user-data/user-data /overlay/config.yaml @@ -364,22 +368,48 @@ build-uki-iso: WORKDIR /build COPY --platform=linux/${ARCH} --keep-own +iso-image-rootfs/rootfs /build/image + # AuroraBoot v0.26.1 silently ignores --output/-d for "dir:" sources on + # both build-iso and build-uki, dropping the ISO at /tmp/auroraboot/*.iso + # regardless. We hoist it into /iso/ ourselves after the run. IF [ "$ARCH" = "arm64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch $ARCH" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + # arm64 UKI ISO is not supported by upstream today; fall through to a + # plain live/installer ISO, matching the previous osbuilder behavior. + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso dir:/build/image --overlay-iso /overlay --arch arm64 ELSE IF [ "$ARCH" = "amd64" ] COPY secure-boot/enrollment/ secure-boot/private-keys/ secure-boot/public-keys/ /keys RUN ls -liah /keys - RUN mkdir /iso + # AuroraBoot's build-uki takes explicit key paths instead of osbuilder's + # bundled -k /keys. All key files live at /keys/* because the three + # secure-boot/* dirs above are flattened into the same target. IF [ "$AUTO_ENROLL_SECUREBOOT_KEYS" = "true" ] - RUN enki --config-dir /config build-uki dir:/build/image --extend-cmdline "$CMDLINE" --overlay-iso /overlay --secure-boot-enroll force -t iso -d /iso -k /keys --boot-branding "$BRANDING" + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-uki dir:/build/image -t iso \ + --extend-cmdline "$CMDLINE" \ + --overlay-iso /overlay \ + --boot-branding "$BRANDING" \ + --public-keys /keys \ + --sb-key /keys/db.key \ + --sb-cert /keys/db.pem \ + --tpm-pcr-private-key /keys/tpm2-pcr-private.pem \ + --secure-boot-enroll force ELSE - RUN enki --config-dir /config build-uki dir:/build/image --extend-cmdline "$CMDLINE" --overlay-iso /overlay -t iso -d /iso -k /keys --boot-branding "$BRANDING" + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-uki dir:/build/image -t iso \ + --extend-cmdline "$CMDLINE" \ + --overlay-iso /overlay \ + --boot-branding "$BRANDING" \ + --public-keys /keys \ + --sb-key /keys/db.key \ + --sb-cert /keys/db.pem \ + --tpm-pcr-private-key /keys/tpm2-pcr-private.pem END END - WORKDIR /iso - RUN mv /iso/*.iso $ISO_NAME.iso + RUN mkdir -p /iso && \ + mv /tmp/auroraboot/*.iso "/iso/$ISO_NAME.iso" SAVE ARTIFACT /iso/* iso: @@ -405,7 +435,12 @@ validate-user-data: build-iso: - FROM --platform=linux/${ARCH} $OSBUILDER_IMAGE + # Switched from quay.io/kairos/osbuilder-tools (archived kairos-io/osbuilder + # + kairos-io/enki) to AuroraBoot, which is the maintained successor. The + # build-iso subcommand accepts a "dir:" source with the same semantics as + # osbuilder's /entrypoint.sh build-iso, so the rootfs preparation path + # above is unchanged; only the final CLI invocation differs. + FROM --platform=linux/${ARCH} $AURORABOOT_IMAGE ENV ISO_NAME=${ISO_NAME} COPY overlay/files-iso/ /overlay/ COPY --if-exists +validate-user-data/user-data /overlay/files-iso/config.yaml @@ -449,17 +484,42 @@ build-iso: rm -f /build/image/opt/spectrocloud/local-ui.tar; \ fi + # AuroraBoot uses Go arch names for both amd64 and arm64 (osbuilder used + # "x86_64" for amd64). --output/--override-name are inert for "dir:" + # sources in v0.26.2 -- the ISO always lands at /tmp/auroraboot/ + # kairos---core--generic-v.iso -- so we + # leave --output default and hoist the produced ISO into /iso/ ourselves. IF [ "$ARCH" = "arm64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch $ARCH" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso dir:/build/image --overlay-iso /overlay --arch arm64 ELSE IF [ "$ARCH" = "amd64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch x86_64" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso dir:/build/image --overlay-iso /overlay --arch amd64 END + RUN mkdir -p /iso && \ + mv /tmp/auroraboot/*.iso "/iso/$ISO_NAME.iso" + + # AuroraBoot v0.26.2's `build-iso` subcommand runs only: + # PrepDirs -> StepCopyCloudConfig -> StepDumpSource -> StepGenISO + # and NEVER calls StepInjectCC. That step is the one that actually copies + # --overlay-iso content onto the finalised ISO tree; its absence means our + # /overlay/... files (Palette-branded /boot/grub2/grub.cfg, user-data, + # cluster config, content bundles, edge_custom_config) silently disappear. + # Empirically verified: the built ISO's /boot/grub2/grub.cfg is + # AuroraBoot's default "Kairos"-branded template, not our overlay's + # "Palette eXtended Kubernetes Edge Installer" version. + # + # Pipeline mode (docker run auroraboot --set ...) invokes StepInjectCC, + # but that adds DinD, container_image loading, and ~100 lines of Earthfile. + # StepInjectCC's actual work is one xorriso command; do it here directly. + # See kairos-io/AuroraBoot pkg/ops/iso.go InjectISO() for the upstream + # equivalent -- same xorriso invocation. + RUN xorriso -indev "/iso/$ISO_NAME.iso" -outdev "/iso/$ISO_NAME.iso" \ + -map /overlay / -boot_image any replay WORKDIR /iso - RUN sha256sum $ISO_NAME.iso > $ISO_NAME.iso.sha256 + RUN sha256sum "$ISO_NAME.iso" > "$ISO_NAME.iso.sha256" SAVE ARTIFACT --keep-ts /iso/* ### UKI targets @@ -1049,6 +1109,7 @@ iso-image: RUN rm -f /usr/bin/luet END COPY overlay/files/ / + IF [ "$IS_CLOUD_IMAGE" = "true" ] COPY cloud-images/workaround/grubmenu.cfg /etc/kairos/branding/grubmenu.cfg COPY cloud-images/workaround/custom-post-reset.yaml /system/oem/custom-post-reset.yaml diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index 1aaa44fc..62b5af0f 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -1,33 +1,56 @@ -search --file --set=root /boot/kernel.xz +search --no-floppy --file --set=root /boot/kernel set default={{DEFAULT_ENTRY}} set timeout=5 set timeout_style=menu set linux=linux set initrd=initrd + +# Video/console parameters are firmware-specific. +# +# BIOS/CSM: vga=795 requests VESA mode 0x31B via INT 10h from the legacy video +# BIOS, giving the kernel a framebuffer console. Unchanged. +# +# UEFI: there is no VBE, so vga= is inert. Combined with nomodeset (which +# blocks the native KMS driver) the kernel ends up with no usable +# framebuffer and the screen stays black. The last console= also wins +# for /dev/console, so console=tty0 must come last or all output goes +# to serial. The baud is pinned because an unqualified console=ttyS0 +# defaults to 9600 while BMC SOL typically runs at 115200. if [ "${grub_platform}" = "efi" ]; then - echo "Please press 't' to show the boot menu on this console" + set consoleparams="console=ttyS0,115200n8 console=tty0" + set videoparams="" +else + set consoleparams="console=tty1 console=ttyS0" + set videoparams="vga=795 nomodeset" fi -set font=($root)/boot/${grub_cpu}/loader/grub2/fonts/unicode.pf2 -if [ -f ${font} ];then - loadfont ${font} + +# loadfont switches GRUB to gfxterm. Some UEFI implementations (observed on +# Supermicro with ATEN BMC) render nothing in that mode, leaving a blank menu, +# so keep the graphical terminal on the BIOS path only. +if [ "${grub_platform}" != "efi" ]; then + set font=($root)/boot/${grub_cpu}/loader/grub2/fonts/unicode.pf2 + if [ -f ${font} ];then + loadfont ${font} + fi fi + menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable $videoparams nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable $videoparams nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install echo Loading initrd... $initrd ($root)/boot/initrd }