add mnnvl pipeline using gb200 nodes

.azure-pipelines/mnnvl-test.yml

@@ -0,0 +1,127 @@
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+# Do not run multi-nodes-test for PR, we can trigger it manually
+pr: none
+
+parameters:
+- name: vmssName
+  type: string
+  default: ''   # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
+- name: hostEntries
+  type: string
+  default: |
+    10.0.5.145 DSM121082302011
+    10.0.5.150 DSM121082302014
+    10.0.5.177 DSM121082301013
+    10.0.5.225 DSM121082301032
+
+variables:
+  - group: mscclpp
+
+jobs:
+- job: MultiNodesTest
+  displayName: Multi nodes test
+  strategy:
+    matrix:
+      cuda13:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
+  pool:
+    name: msccl-ci-gb200
+  container:
+    image: $[ variables['containerImage'] ]
+
+  steps:
+  - task: Bash@3
+    displayName: Node connectivity check
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -x
+        echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
+        while IFS= read -r line; do
+          ip=$(awk 'NF>=1{print $1}' <<< "$line")
+          [ -z "$ip" ] && continue
+          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
+            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
+        done <<< "${{ parameters.hostEntries }}"
+
+  - task: Bash@3
+    displayName: Add HostEntry
+    inputs:
+      targetType: 'inline'
+      script: |
+        while IFS= read -r line; do
+          [ -z "$line" ] && continue
+          if ! grep -qxF "$line" /etc/hosts; then
+            echo "Adding to /etc/hosts: $line"
+            echo "$line" | sudo tee -a /etc/hosts
+          else
+            echo "Entry already exists: $line"
+          fi
+        done <<< "${{ parameters.hostEntries }}"
+
+  - task: Bash@3
+    displayName: Generate deploy files
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -e
+        DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
+        # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the
+        # names resolve inside the container too (the DSM hostnames are only in
+        # the agent's /etc/hosts). Bump to more nodes by parsing further lines.
+        NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
+        NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
+
+        echo "Host ${NODE0}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no
+        Host ${NODE1}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
+
+        printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+
+        printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
+
+        # Publish the head node so the run steps can target it without hardcoding.
+        echo "##vso[task.setvariable variable=headNode]${NODE0}"
+
+  - template: templates/deploy.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+      pilot: true
+      resourceGroup: mscclpp
+      gpuArch: '100a'
+      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
+
+  - template: templates/ut-gb200.yml
+    parameters:
+      user: azhpcuser
+      headNode: $(headNode)
+      hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
+      gpuArch: '100a'
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: PyBench
+      displayName: Run Collective Benchmarks
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
+      remoteScript: |
+        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory

.azure-pipelines/multi-nodes-test.yml

@@ -122,4 +122,4 @@ jobs:
     parameters:
       subscription: mscclpp-ci-h100
       vmssName: ${{ parameters.vmssName }}
-      resourceGroup: mscclpp
+      resourceGroup: mscclpp

.azure-pipelines/templates/deploy.yml

@@ -6,6 +6,9 @@ parameters:
 - name: resourceGroup
   type: string
   default: mscclpp
+- name: pilot
+  type: boolean
+  default: false
 # Build parameters
 - name: platform
   type: string
@@ -136,15 +139,16 @@ steps:
       sudo apt-get update -y
       sudo apt-get install pssh -y
 
-- task: AzureCLI@2
-  name: StartVMSS
-  displayName: Start VMSS
-  inputs:
-    azureSubscription: ${{ parameters.subscription }}
-    scriptType: bash
-    scriptLocation: inlineScript
-    inlineScript: |
-      az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }}
+- ${{ if not(parameters.pilot) }}:
+  - task: AzureCLI@2
+    name: StartVMSS
+    displayName: Start VMSS
+    inputs:
+      azureSubscription: ${{ parameters.subscription }}
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }}
 
 # 4. Deploy test environment
 - task: Bash@3
@@ -153,5 +157,5 @@ steps:
   inputs:
     targetType: filePath
     filePath: test/deploy/deploy.sh
-    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }}
+    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} ${{ parameters.pilot }}
     workingDirectory: '$(System.DefaultWorkingDirectory)'

.azure-pipelines/templates/ut-gb200.yml

@@ -0,0 +1,116 @@
+parameters:
+- name: user
+  type: string
+  default: 'azhpcuser'
+# Head node used to launch mpirun (mnnvl-test.yml publishes this as $(headNode)).
+- name: headNode
+  type: string
+# Agent-side hostfile listing every node (user@ip per line). run-remote.sh uses
+# it to build the tests on all nodes in parallel.
+- name: hostfile
+  type: string
+  default: '$(System.DefaultWorkingDirectory)/test/deploy/hostfile'
+# Container-side hostfile passed to mpirun (one ip per line).
+- name: gpuArch
+  type: string
+  default: '100a'
+# Single-node rank ladder for mp_unit_tests. Space-separated; each value is one
+# mpirun -np run on the head node. GB200 nodes have 4 GPUs, so 2 and 4 exercise
+# the intra-node paths (IPC / NVLink). Every value must be >= 2 (the Prelim test
+# asserts worldSize >= 2). The communicator/channel/executor tests only have a
+# working transport intra-node in a no-IB build, so they must run single-node.
+- name: singleNodeNps
+  type: string
+  default: '2 4'
+# Negative substring filter for the single-node runs. Comma-separated patterns
+# are all excluded. -Ib skips the InfiniBand tests (no IB on MNNVL).
+- name: filter
+  type: string
+  default: '-Ib'
+# Whether to also run a cross-node mp_unit_tests pass. It validates the
+# multi-node TcpBootstrap rendezvous over the routable subnet -- the path unique
+# to the MNNVL topology -- using only the tests that work cross-node without IB.
+- name: runCrossNodeBootstrap
+  type: boolean
+  default: true
+# Negative filter for the cross-node pass. Excludes every fixture that needs a
+# cross-node data transport (Communicator / Executor / *Channel / Semaphore /
+# Ib), which a no-IB build lacks between nodes and which otherwise deadlocks on
+# the mesh barrier. Also excludes TimeoutWithId, whose 100ms window on a 1s
+# timeout is too tight for remote ranks over TCP. What remains: BootstrapTest.*
+# and MultiProcessTest.* (Prelim / HostName / HostHash).
+- name: crossNodeFilter
+  type: string
+  default: '-Communicator,Executor,Ib,MemoryChannel,PortChannel,SemaphorePerf,SwitchChannel,TimeoutWithId'
+# TcpBootstrap rendezvous port. The bootstrap root address is set to
+# <headNode>:<bootstrapPort> so ranks on other nodes reach rank 0 over the
+# routable subnet. The mp_unit_tests default (127.0.0.1) only works
+# single-node and makes BootstrapTest.*IpPortPair time out across nodes.
+- name: bootstrapPort
+  type: number
+  default: 50053
+# Whether to also run the single-process unit_tests binary.
+- name: runUnitTests
+  type: boolean
+  default: true
+
+steps:
+# Rebuild the C++ tests natively on every node so the binaries are aarch64.
+# The x86-64 CMakeCache.txt shipped from the agent is discarded (rm -rf build).
+- template: run-remote-task.yml
+  parameters:
+    name: BuildGb200Tests
+    displayName: Build unit tests on GB200 nodes
+    runRemoteArgs: '--hostfile ${{ parameters.hostfile }} --user ${{ parameters.user }}'
+    remoteScript: |
+      rm -rf build
+      mkdir -p build && cd build
+      cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DMSCCLPP_BYPASS_GPU_CHECK=ON \
+        -DMSCCLPP_USE_CUDA=ON \
+        -DMSCCLPP_BUILD_TESTS=ON \
+        -DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }} \
+        -DMSCCLPP_USE_IB=OFF ..
+      make -j unit_tests mp_unit_tests
+
+- ${{ if parameters.runUnitTests }}:
+  - template: run-remote-task.yml
+    parameters:
+      name: Gb200UnitTests
+      displayName: Run mscclpp unit tests
+      runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+      remoteScript: |
+        ./build/bin/unit_tests
+
+# Run mp_unit_tests single-node on the head node across the rank ladder. No
+# hostfile / ip_port needed: all ranks are local, so loopback bootstrap works and
+# every fixture has a working intra-node transport (IPC / NVLink).
+- template: run-remote-task.yml
+  parameters:
+    name: Gb200MpUnitTests
+    displayName: Run mscclpp multi-process unit tests (single-node)
+    runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+    remoteScript: |
+      for np in ${{ parameters.singleNodeNps }}; do
+        echo "=== mp_unit_tests -np ${np} (single-node) ==="
+        mpirun --allow-run-as-root -tag-output --bind-to numa -np ${np} \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --filter=${{ parameters.filter }}
+      done
+
+# Cross-node pass: validate the multi-node TcpBootstrap rendezvous. mpirun spans
+# both nodes via the container hostfile; --ip_port points every rank at rank 0 on
+# the head node (the mp_unit_tests default of 127.0.0.1 only works single-node
+# and makes the *IpPortPair tests time out across nodes). crossNodeFilter keeps
+# this to the fixtures that work without cross-node IB.
+- ${{ if parameters.runCrossNodeBootstrap }}:
+  - template: run-remote-task.yml
+    parameters:
+      name: Gb200MpBootstrapMultiNode
+      displayName: Run mscclpp multi-process bootstrap tests (cross-node)
+      runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+      remoteScript: |
+        mpirun --allow-run-as-root -tag-output --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --ip_port=${{ parameters.headNode }}:${{ parameters.bootstrapPort }} --filter=${{ parameters.crossNodeFilter }}

docker/build.sh

@@ -96,17 +96,17 @@ echo ""
 echo "    docker tag ${TAG_BASE_DEV} ${GHCR_TAG_BASE_DEV_ARCH} && \\"
 echo "    docker push ${GHCR_TAG_BASE_DEV_ARCH}"
 echo ""
-echo "2. Create or update the multi-arch manifest:"
-echo ""
-echo "   If \`${GHCR_TAG_BASE_DEV}\` already exists (adding another arch):"
+echo "2. (Re)create the multi-arch manifest — ALWAYS list EVERY arch tag you have"
+echo "   published for this target. \`imagetools create\` REPLACES the manifest,"
+echo "   so omitting an arch silently drops it:"
 echo ""
 echo "    docker buildx imagetools create \\"
 echo "        --tag ${GHCR_TAG_BASE_DEV} \\"
-echo "        --append ${GHCR_TAG_BASE_DEV_ARCH}"
+echo "        ${GHCR}:base-dev-${TARGET}-x86_64 \\"
+echo "        ${GHCR}:base-dev-${TARGET}-aarch64"
 echo ""
-echo "   If \`${GHCR_TAG_BASE_DEV}\` does not exist yet:"
+echo "   (Include only the arch tags that actually exist for this target — e.g."
+echo "    ROCm targets are x86_64-only. Verify afterward:)"
 echo ""
-echo "    docker buildx imagetools create \\"
-echo "        --tag ${GHCR_TAG_BASE_DEV} \\"
-echo "        ${GHCR_TAG_BASE_DEV_ARCH}"
+echo "    docker buildx imagetools inspect ${GHCR_TAG_BASE_DEV}"
 echo ""

python/mscclpp_benchmark/bench_collective.py

@@ -199,6 +199,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl
             "default_allreduce_nvls_packet",
             max_message_size=512 * 1024,
             max_nblocks=16,
+            supported_skus=("H100", "GB300", "GB200"),
             requires_nvls=True,
         ),
         CandidateSpec(
@@ -231,6 +232,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl
             CandidateSpec(
                 "default_allreduce_nvls_zero_copy",
                 max_nblocks=32,
+                supported_skus=("H100", "GB300", "GB200"),
                 requires_nvls=True,
                 requires_symmetric_memory=True,
             ),

python/mscclpp_benchmark/tuning_config.py

@@ -10,7 +10,7 @@
 from pathlib import Path
 from typing import Any
 
-_KNOWN_GPU_SKUS = ("GB300", "MI300X", "H100", "A100")
+_KNOWN_GPU_SKUS = ("GB300", "GB200", "MI300X", "H100", "A100")
 
 
 @dataclass(frozen=True)