From 2083df3d996fd578cab9751486a32e1ee9ab7cc4 Mon Sep 17 00:00:00 2001
From: empyreus <rjsouza1995@gmail.com>
Date: Fri, 17 Jul 2026 09:31:45 -0700
Subject: [PATCH 01/51] initial work

---
 .azure-pipelines/mnnvl-test.yml           | 122 ++++++++++++++++++++++
 .azure-pipelines/templates/deploy.yml     |  24 +++--
 .azure-pipelines/templates/mnnvl-test.yml |   0
 test/deploy/deploy.sh                     |   1 +
 4 files changed, 137 insertions(+), 10 deletions(-)
 create mode 100644 .azure-pipelines/mnnvl-test.yml
 create mode 100644 .azure-pipelines/templates/mnnvl-test.yml

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
new file mode 100644
index 000000000..684fd56fe
--- /dev/null
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -0,0 +1,122 @@
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+# Do not run multi-nodes-test for PR, we can trigger it manually
+pr: none
+
+parameters:
+- name: vmssName
+  type: string
+  default: mscclpp-h100-multinode-ci 
+- name: hostEntries
+  type: string
+  default: |
+    10.0.5.145 DSM121082302011
+    10.0.5.150 DSM121082302014
+    10.0.5.177 DSM121082301013
+    10.0.5.225 DSM121082301032
+
+jobs:
+- job: MultiNodesTest
+  displayName: Multi nodes test
+  strategy:
+    matrix:
+      cuda12:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
+      cuda13:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
+  pool:
+    name: mscclpp-multi-node
+  container:
+    image: $[ variables['containerImage'] ]
+
+  steps:
+  - task: Bash@3
+    displayName: Add HostEntry
+    inputs:
+      targetType: 'inline'
+      script: |
+        while IFS= read -r line; do
+          [ -z "$line" ] && continue
+          if ! grep -qxF "$line" /etc/hosts; then
+            echo "Adding to /etc/hosts: $line"
+            echo "$line" | sudo tee -a /etc/hosts
+          else
+            echo "Entry already exists: $line"
+          fi
+        done <<< "${{ parameters.hostEntries }}"
+
+  - task: Bash@3
+    displayName: Generate deploy files
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -e
+        VMSS="${{ parameters.vmssName }}"
+        DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
+        NODE0="${VMSS}000000"
+        NODE1="${VMSS}000001"
+
+        echo "Host ${NODE0}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no
+        Host ${NODE1}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
+
+        printf '%s\n%s\n' "azureuser@${NODE0}" "azureuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+
+        printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
+
+  - template: templates/deploy.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+      pilot: true
+      resourceGroup: mscclpp
+      gpuArch: '100a'
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMscclppTest
+      displayName: Run multi-nodes mscclpp-test
+      continueOnError: true
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMultiNodeUnitTest
+      displayName: Run multi-nodes unit tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMultiNodePythonTests
+      displayName: Run multi-nodes python tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh pytests
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMultiNodePythonBenchmark
+      displayName: Run multi-nodes python benchmark
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark
diff --git a/.azure-pipelines/templates/deploy.yml b/.azure-pipelines/templates/deploy.yml
index 9eb46d8b9..97ecb8374 100644
--- a/.azure-pipelines/templates/deploy.yml
+++ b/.azure-pipelines/templates/deploy.yml
@@ -6,6 +6,9 @@ parameters:
 - name: resourceGroup
   type: string
   default: mscclpp
+- name: pilot
+  type: boolean
+  default: false
 # Build parameters
 - name: platform
   type: string
@@ -136,15 +139,16 @@ steps:
       sudo apt-get update -y
       sudo apt-get install pssh -y
 
-- task: AzureCLI@2
-  name: StartVMSS
-  displayName: Start VMSS
-  inputs:
-    azureSubscription: ${{ parameters.subscription }}
-    scriptType: bash
-    scriptLocation: inlineScript
-    inlineScript: |
-      az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }}
+- ${{ if not(parameters.pilot) }}:
+  - task: AzureCLI@2
+    name: StartVMSS
+    displayName: Start VMSS
+    inputs:
+      azureSubscription: ${{ parameters.subscription }}
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }}
 
 # 4. Deploy test environment
 - task: Bash@3
@@ -153,5 +157,5 @@ steps:
   inputs:
     targetType: filePath
     filePath: test/deploy/deploy.sh
-    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }}
+    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} ${{ parameters.pilot }}
     workingDirectory: '$(System.DefaultWorkingDirectory)'
diff --git a/.azure-pipelines/templates/mnnvl-test.yml b/.azure-pipelines/templates/mnnvl-test.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh
index 02fe4fd25..0ab8f0120 100644
--- a/test/deploy/deploy.sh
+++ b/test/deploy/deploy.sh
@@ -18,6 +18,7 @@ IB_ENVIRONMENT="${2:-true}"
 PLATFORM="${3:-cuda}"
 CONTAINER_NAME="${4:-mscclpp-test}"
 SGLANG_IMAGE="${5:-lmsysorg/sglang:latest}"
+PILOT="$(echo "${6:-false}" | tr '[:upper:]' '[:lower:]')"
 
 KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
 ROOT_DIR="${SYSTEM_DEFAULTWORKINGDIRECTORY}/"

From db96d1ce4aaf8cb1d9999607fce601ed858af3a5 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 31 Jul 2026 11:46:38 -0700
Subject: [PATCH 02/51] replace multi-node to test

---
 .azure-pipelines/multi-nodes-test.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index c4e27be47..684fd56fe 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -17,12 +17,14 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: mscclpp-h100-multinode-ci
+  default: mscclpp-h100-multinode-ci 
 - name: hostEntries
   type: string
   default: |
-    10.0.0.5 mscclpp-h100-multinode-ci000000
-    10.0.0.4 mscclpp-h100-multinode-ci000001
+    10.0.5.145 DSM121082302011
+    10.0.5.150 DSM121082302014
+    10.0.5.177 DSM121082301013
+    10.0.5.225 DSM121082301032
 
 jobs:
 - job: MultiNodesTest
@@ -82,8 +84,9 @@ jobs:
     parameters:
       subscription: mscclpp-ci-h100
       vmssName: ${{ parameters.vmssName }}
+      pilot: true
       resourceGroup: mscclpp
-      gpuArch: '90'
+      gpuArch: '100a'
 
   - template: templates/run-remote-task.yml
     parameters:
@@ -117,9 +120,3 @@ jobs:
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark
-
-  - template: templates/stop.yml
-    parameters:
-      subscription: mscclpp-ci-h100
-      vmssName: ${{ parameters.vmssName }}
-      resourceGroup: mscclpp

From db20006123d422009b6326e6275ca9c11ec9f337 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 31 Jul 2026 11:47:14 -0700
Subject: [PATCH 03/51] update deploy and run-remote

---
 test/deploy/deploy.sh     | 35 +++++++++++++++++++++++------------
 test/deploy/run-remote.sh | 13 ++++++++++++-
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh
index 0ab8f0120..321f5bb74 100644
--- a/test/deploy/deploy.sh
+++ b/test/deploy/deploy.sh
@@ -32,12 +32,23 @@ fi
 
 SSH_OPTION="StrictHostKeyChecking=no"
 
+# CI-agent -> node SSH args. Optionally route through a jumpbox via ProxyCommand
+# (opt-in through SSH_PROXY_JUMP; SSH_PROXY_KEY selects a separate jump key).
+# When the vars are unset there is no proxy, so pipelines that don't set them
+# (e.g. multi-nodes-test.yml) keep using direct SSH unchanged.
+SSH_EXTRA_ARGS="-i ${KeyFilePath}"
+if [ -n "${SSH_PROXY_JUMP:-}" ]; then
+  PROXY_KEY_OPT=""
+  [ -n "${SSH_PROXY_KEY:-}" ] && PROXY_KEY_OPT="-i ${SSH_PROXY_KEY} "
+  SSH_EXTRA_ARGS="${SSH_EXTRA_ARGS} -o ProxyCommand=\"ssh ${PROXY_KEY_OPT}-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p ${SSH_PROXY_JUMP}\""
+fi
+
 chmod 400 ${KeyFilePath}
 ssh-keygen -t rsa -f sshkey -P ""
 
 while true; do
   set +e
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "hostname"
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "hostname"
   if [ $? -eq 0 ]; then
     break
   fi
@@ -46,21 +57,21 @@ while true; do
 done
 set -e
 
-parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "sudo rm -rf ${DST_DIR}"
+parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "sudo rm -rf ${DST_DIR}"
 tar czf /tmp/mscclpp.tar.gz -C ${ROOT_DIR} .
-parallel-scp -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION /tmp/mscclpp.tar.gz /tmp/mscclpp.tar.gz
-parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+parallel-scp -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION /tmp/mscclpp.tar.gz /tmp/mscclpp.tar.gz
+parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
   "sudo mkdir -p ${DST_DIR} && sudo tar xzf /tmp/mscclpp.tar.gz -C ${DST_DIR} && sudo rm -f /tmp/mscclpp.tar.gz"
 rm -f /tmp/mscclpp.tar.gz
 
 if [ "${PLATFORM}" == "rocm" ]; then
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "sudo modprobe amdgpu"
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "sudo modprobe amdgpu"
 fi
 
 # Install GDRCopy kernel module on host VMs (CUDA only)
 GDRCOPY_VERSION="2.5.2"
 if [ "${PLATFORM}" == "cuda" ]; then
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
     "if lsmod | grep -q gdrdrv; then
       echo 'gdrdrv module already loaded'
     else
@@ -77,14 +88,14 @@ fi
 
 if [ "${CONTAINER_NAME}" == "sglang-mscclpp-test" ]; then
   # force to pull the latest image
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
     "sudo docker pull ${SGLANG_IMAGE}"
 
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
     "sudo docker run --rm -itd --name=${CONTAINER_NAME} --privileged --net=host --ipc=host --gpus=all -w /root -v ${DST_DIR}:/root/mscclpp --entrypoint /bin/bash ${SGLANG_IMAGE}"
 else
   # force to pull the latest image
-  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+  parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
     "sudo docker pull ${CONTAINERIMAGE}"
 
   # Set GPU passthrough flags based on platform
@@ -95,18 +106,18 @@ else
 
   if [ "${IB_ENVIRONMENT}" == "true" ]; then
     # InfiniBand: use --privileged for RDMA device access
-    parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+    parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
       "sudo docker run --rm -itd --privileged --net=host --ipc=host ${LAUNCH_OPTION} \
       -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \
       --entrypoint /bin/bash ${CONTAINERIMAGE}"
   else
     # Non-IB: grant SYS_ADMIN and disable seccomp instead of full --privileged
-    parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+    parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
       "sudo docker run --rm -itd --net=host --ipc=host ${LAUNCH_OPTION} --cap-add=SYS_ADMIN --security-opt seccomp=unconfined \
       -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \
       --entrypoint /bin/bash ${CONTAINERIMAGE}"
   fi
 fi
 
-parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \
+parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
   "sudo docker exec -t --user root ${CONTAINER_NAME} bash '/root/mscclpp/test/deploy/setup.sh' ${PLATFORM}"
diff --git a/test/deploy/run-remote.sh b/test/deploy/run-remote.sh
index 9607664fc..2e2f7732a 100755
--- a/test/deploy/run-remote.sh
+++ b/test/deploy/run-remote.sh
@@ -20,6 +20,17 @@ HOSTFILE="${SCRIPT_DIR}/hostfile_ci"
 SSH_OPTION="StrictHostKeyChecking=no"
 KeyFilePath="${SSHKEYFILE_SECUREFILEPATH}"
 
+# CI-agent -> node SSH args. Optionally route through a jumpbox via ProxyCommand
+# (opt-in through SSH_PROXY_JUMP; SSH_PROXY_KEY selects a separate jump key).
+# When the vars are unset there is no proxy, so pipelines that don't set them
+# keep using direct SSH unchanged.
+SSH_EXTRA_ARGS="-i ${KeyFilePath}"
+if [ -n "${SSH_PROXY_JUMP:-}" ]; then
+    PROXY_KEY_OPT=""
+    [ -n "${SSH_PROXY_KEY:-}" ] && PROXY_KEY_OPT="-i ${SSH_PROXY_KEY} "
+    SSH_EXTRA_ARGS="${SSH_EXTRA_ARGS} -o ProxyCommand=\"ssh ${PROXY_KEY_OPT}-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p ${SSH_PROXY_JUMP}\""
+fi
+
 USE_DOCKER=true
 USE_LOG=true
 TARGET_HOST=""
@@ -95,7 +106,7 @@ PSSH_COMMON=(
     -t 0
     "${PSSH_TARGET_ARGS[@]}"
     "${PSSH_USER_ARGS[@]}"
-    -x "-i ${KeyFilePath}"
+    -x "${SSH_EXTRA_ARGS}"
     -O "$SSH_OPTION"
 )
 

From a21bd4127a43f7c3743ac4a02b225afc917849a8 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 31 Jul 2026 13:43:20 -0700
Subject: [PATCH 04/51] test deploy

---
 .azure-pipelines/mnnvl-test.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 684fd56fe..76233f5e5 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -17,7 +17,7 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: mscclpp-h100-multinode-ci 
+  default: mscclpp-h100-multinode-ci # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
 - name: hostEntries
   type: string
   default: |
@@ -62,10 +62,11 @@ jobs:
       targetType: 'inline'
       script: |
         set -e
-        VMSS="${{ parameters.vmssName }}"
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
-        NODE0="${VMSS}000000"
-        NODE1="${VMSS}000001"
+        # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
+        # Bump to more nodes later by parsing further lines the same way.
+        NODE0=$(awk 'NF>=2{print $2; exit}'           <<< "${{ parameters.hostEntries }}")
+        NODE1=$(awk 'NF>=2{c++} c==2{print $2; exit}' <<< "${{ parameters.hostEntries }}")
 
         echo "Host ${NODE0}
           Port 22345
@@ -80,6 +81,9 @@ jobs:
 
         printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
 
+        # Publish the head node so the run steps can target it without hardcoding.
+        echo "##vso[task.setvariable variable=headNode]${NODE0}"
+
   - template: templates/deploy.yml
     parameters:
       subscription: mscclpp-ci-h100
@@ -93,7 +97,7 @@ jobs:
       name: RunMscclppTest
       displayName: Run multi-nodes mscclpp-test
       continueOnError: true
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
 
@@ -101,7 +105,7 @@ jobs:
     parameters:
       name: RunMultiNodeUnitTest
       displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
 
@@ -109,7 +113,7 @@ jobs:
     parameters:
       name: RunMultiNodePythonTests
       displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh pytests
 
@@ -117,6 +121,6 @@ jobs:
     parameters:
       name: RunMultiNodePythonBenchmark
       displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark

From 72333e0f9a408d1bf69ffc5131b7826b543fcf93 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 31 Jul 2026 14:00:52 -0700
Subject: [PATCH 05/51] test

---
 .azure-pipelines/mnnvl-test.yml       |  2 +-
 .azure-pipelines/multi-nodes-test.yml | 20 ++++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 76233f5e5..97f536e93 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -17,7 +17,7 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: mscclpp-h100-multinode-ci # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
+  default: ''   # unused on pilot (StartVMSS is gated off); kept only to satisfy deploy.yml # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
 - name: hostEntries
   type: string
   default: |
diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 684fd56fe..97f536e93 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -17,7 +17,7 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: mscclpp-h100-multinode-ci 
+  default: ''   # unused on pilot (StartVMSS is gated off); kept only to satisfy deploy.yml # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
 - name: hostEntries
   type: string
   default: |
@@ -62,10 +62,11 @@ jobs:
       targetType: 'inline'
       script: |
         set -e
-        VMSS="${{ parameters.vmssName }}"
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
-        NODE0="${VMSS}000000"
-        NODE1="${VMSS}000001"
+        # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
+        # Bump to more nodes later by parsing further lines the same way.
+        NODE0=$(awk 'NF>=2{print $2; exit}'           <<< "${{ parameters.hostEntries }}")
+        NODE1=$(awk 'NF>=2{c++} c==2{print $2; exit}' <<< "${{ parameters.hostEntries }}")
 
         echo "Host ${NODE0}
           Port 22345
@@ -80,6 +81,9 @@ jobs:
 
         printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
 
+        # Publish the head node so the run steps can target it without hardcoding.
+        echo "##vso[task.setvariable variable=headNode]${NODE0}"
+
   - template: templates/deploy.yml
     parameters:
       subscription: mscclpp-ci-h100
@@ -93,7 +97,7 @@ jobs:
       name: RunMscclppTest
       displayName: Run multi-nodes mscclpp-test
       continueOnError: true
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
 
@@ -101,7 +105,7 @@ jobs:
     parameters:
       name: RunMultiNodeUnitTest
       displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
 
@@ -109,7 +113,7 @@ jobs:
     parameters:
       name: RunMultiNodePythonTests
       displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh pytests
 
@@ -117,6 +121,6 @@ jobs:
     parameters:
       name: RunMultiNodePythonBenchmark
       displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark

From 7cb76618515abaf8245d09cc4e7670c4bb93c14e Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Mon, 3 Aug 2026 13:55:41 -0700
Subject: [PATCH 06/51] add variable group

---
 .azure-pipelines/mnnvl-test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 97f536e93..d7ad181ab 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -17,7 +17,7 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: ''   # unused on pilot (StartVMSS is gated off); kept only to satisfy deploy.yml # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
+  default: ''   # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
 - name: hostEntries
   type: string
   default: |
@@ -26,6 +26,13 @@ parameters:
     10.0.5.177 DSM121082301013
     10.0.5.225 DSM121082301032
 
+variables:
+  - group: mscclpp
+  - name: SSH_PROXY_JUMP
+    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
+  - name: SSH_PROXY_KEY
+    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
+
 jobs:
 - job: MultiNodesTest
   displayName: Multi nodes test

From c5b11978ef51bf56f08e43798c4eb4cf460859dc Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Mon, 3 Aug 2026 13:56:31 -0700
Subject: [PATCH 07/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 97f536e93..d7ad181ab 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -17,7 +17,7 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: ''   # unused on pilot (StartVMSS is gated off); kept only to satisfy deploy.yml # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
+  default: ''   # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
 - name: hostEntries
   type: string
   default: |
@@ -26,6 +26,13 @@ parameters:
     10.0.5.177 DSM121082301013
     10.0.5.225 DSM121082301032
 
+variables:
+  - group: mscclpp
+  - name: SSH_PROXY_JUMP
+    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
+  - name: SSH_PROXY_KEY
+    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
+
 jobs:
 - job: MultiNodesTest
   displayName: Multi nodes test

From e121f2a98da3896298fe93501a41ea1c09d9adfe Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Mon, 3 Aug 2026 15:02:46 -0700
Subject: [PATCH 08/51] add diag

---
 .azure-pipelines/multi-nodes-test.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index d7ad181ab..1109fc731 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -48,6 +48,17 @@ jobs:
     image: $[ variables['containerImage'] ]
 
   steps:
+  - task: Bash@3
+    displayName: Jumpbox connectivity check
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -x
+        JB="$(GB200-JUMPBOX-IP)"
+        echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
+        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
+          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+
   - task: Bash@3
     displayName: Add HostEntry
     inputs:

From cd61b32bfd6acc25d23342a611e9da0e82974f3a Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 13:52:01 -0700
Subject: [PATCH 09/51] update agent pool

---
 .azure-pipelines/mnnvl-test.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index d7ad181ab..198aa4dca 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -43,11 +43,22 @@ jobs:
       cuda13:
         containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
-    name: mscclpp-multi-node
+    name: msccl-ci-gb200
   container:
     image: $[ variables['containerImage'] ]
 
   steps:
+  - task: Bash@3
+    displayName: Jumpbox connectivity check
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -x
+        JB="$(GB200-JUMPBOX-IP)"
+        echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
+        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
+          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+
   - task: Bash@3
     displayName: Add HostEntry
     inputs:

From 49a1bda818f593043d98fb900d120ef2deb8a85e Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 13:53:23 -0700
Subject: [PATCH 10/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 1109fc731..198aa4dca 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -43,7 +43,7 @@ jobs:
       cuda13:
         containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
-    name: mscclpp-multi-node
+    name: msccl-ci-gb200
   container:
     image: $[ variables['containerImage'] ]
 

From 2e64ee087ea2ebeddd140e8cc90a24377c220621 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 17:02:42 -0700
Subject: [PATCH 11/51] remove jumpbox

---
 .azure-pipelines/mnnvl-test.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 198aa4dca..5c9881517 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -28,10 +28,6 @@ parameters:
 
 variables:
   - group: mscclpp
-  - name: SSH_PROXY_JUMP
-    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
-  - name: SSH_PROXY_KEY
-    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -49,15 +45,18 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Jumpbox connectivity check
+    displayName: Node connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
-        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
-          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+        while IFS= read -r line; do
+          ip=$(awk 'NF>=1{print $1}' <<< "$line")
+          [ -z "$ip" ] && continue
+          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
+            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
+        done <<< "${{ parameters.hostEntries }}"
 
   - task: Bash@3
     displayName: Add HostEntry

From 8bd5ae8f710cb069a0ab670b0a76d97e599ec8e6 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 17:03:01 -0700
Subject: [PATCH 12/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 198aa4dca..5c9881517 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -28,10 +28,6 @@ parameters:
 
 variables:
   - group: mscclpp
-  - name: SSH_PROXY_JUMP
-    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
-  - name: SSH_PROXY_KEY
-    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -49,15 +45,18 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Jumpbox connectivity check
+    displayName: Node connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
-        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
-          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+        while IFS= read -r line; do
+          ip=$(awk 'NF>=1{print $1}' <<< "$line")
+          [ -z "$ip" ] && continue
+          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
+            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
+        done <<< "${{ parameters.hostEntries }}"
 
   - task: Bash@3
     displayName: Add HostEntry

From bcbeead01f40da8498ac45bae396aa0b73a19ebd Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 17:44:38 -0700
Subject: [PATCH 13/51] add jumpbox

---
 .azure-pipelines/mnnvl-test.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 5c9881517..198aa4dca 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -28,6 +28,10 @@ parameters:
 
 variables:
   - group: mscclpp
+  - name: SSH_PROXY_JUMP
+    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
+  - name: SSH_PROXY_KEY
+    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -45,18 +49,15 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Node connectivity check
+    displayName: Jumpbox connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
+        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        while IFS= read -r line; do
-          ip=$(awk 'NF>=1{print $1}' <<< "$line")
-          [ -z "$ip" ] && continue
-          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
-            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
-        done <<< "${{ parameters.hostEntries }}"
+        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
+          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
 
   - task: Bash@3
     displayName: Add HostEntry

From 85333b168194c7ede7218bb7eaa4c51a0a336969 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 17:45:00 -0700
Subject: [PATCH 14/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 5c9881517..198aa4dca 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -28,6 +28,10 @@ parameters:
 
 variables:
   - group: mscclpp
+  - name: SSH_PROXY_JUMP
+    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
+  - name: SSH_PROXY_KEY
+    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -45,18 +49,15 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Node connectivity check
+    displayName: Jumpbox connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
+        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        while IFS= read -r line; do
-          ip=$(awk 'NF>=1{print $1}' <<< "$line")
-          [ -z "$ip" ] && continue
-          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
-            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
-        done <<< "${{ parameters.hostEntries }}"
+        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
+          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
 
   - task: Bash@3
     displayName: Add HostEntry

From 810cd19040e7cfec484505d9b3ce2370b9857e80 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 17:54:36 -0700
Subject: [PATCH 15/51] remove jumpbox

---
 .azure-pipelines/mnnvl-test.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 198aa4dca..5c9881517 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -28,10 +28,6 @@ parameters:
 
 variables:
   - group: mscclpp
-  - name: SSH_PROXY_JUMP
-    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
-  - name: SSH_PROXY_KEY
-    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -49,15 +45,18 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Jumpbox connectivity check
+    displayName: Node connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
-        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
-          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+        while IFS= read -r line; do
+          ip=$(awk 'NF>=1{print $1}' <<< "$line")
+          [ -z "$ip" ] && continue
+          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
+            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
+        done <<< "${{ parameters.hostEntries }}"
 
   - task: Bash@3
     displayName: Add HostEntry

From 030a848f77bac55814f1400e20eb22425a63fd8d Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 18:15:55 -0700
Subject: [PATCH 16/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 198aa4dca..5c9881517 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -28,10 +28,6 @@ parameters:
 
 variables:
   - group: mscclpp
-  - name: SSH_PROXY_JUMP
-    value: $(GB200-JUMPBOX-USER)@$(GB200-JUMPBOX-IP)
-  - name: SSH_PROXY_KEY
-    value: $(SshKeyFile.secureFilePath)   # same mscclpp.pem key downloaded by deploy.yml
 
 jobs:
 - job: MultiNodesTest
@@ -49,15 +45,18 @@ jobs:
 
   steps:
   - task: Bash@3
-    displayName: Jumpbox connectivity check
+    displayName: Node connectivity check
     inputs:
       targetType: 'inline'
       script: |
         set -x
-        JB="$(GB200-JUMPBOX-IP)"
         echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        timeout 10 bash -c "cat < /dev/null > /dev/tcp/${JB}/22" \
-          && echo "port 22 OPEN" || echo "port 22 UNREACHABLE"
+        while IFS= read -r line; do
+          ip=$(awk 'NF>=1{print $1}' <<< "$line")
+          [ -z "$ip" ] && continue
+          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
+            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
+        done <<< "${{ parameters.hostEntries }}"
 
   - task: Bash@3
     displayName: Add HostEntry

From e67085fcb367fcc820eff67636a06514c020664a Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 18:46:15 -0700
Subject: [PATCH 17/51] fix user name

---
 .azure-pipelines/mnnvl-test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 5c9881517..4bffbc51a 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -94,7 +94,7 @@ jobs:
           IdentityFile /root/mscclpp/sshkey
           StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
 
-        printf '%s\n%s\n' "azureuser@${NODE0}" "azureuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+        printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
 
         printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
 
@@ -114,7 +114,7 @@ jobs:
       name: RunMscclppTest
       displayName: Run multi-nodes mscclpp-test
       continueOnError: true
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
 
@@ -122,7 +122,7 @@ jobs:
     parameters:
       name: RunMultiNodeUnitTest
       displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
 
@@ -130,7 +130,7 @@ jobs:
     parameters:
       name: RunMultiNodePythonTests
       displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh pytests
 
@@ -138,6 +138,6 @@ jobs:
     parameters:
       name: RunMultiNodePythonBenchmark
       displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark

From 09df80814418d1c84878b028349656b8f32e9c42 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 4 Aug 2026 18:46:36 -0700
Subject: [PATCH 18/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 5c9881517..4bffbc51a 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -94,7 +94,7 @@ jobs:
           IdentityFile /root/mscclpp/sshkey
           StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
 
-        printf '%s\n%s\n' "azureuser@${NODE0}" "azureuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+        printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
 
         printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
 
@@ -114,7 +114,7 @@ jobs:
       name: RunMscclppTest
       displayName: Run multi-nodes mscclpp-test
       continueOnError: true
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
 
@@ -122,7 +122,7 @@ jobs:
     parameters:
       name: RunMultiNodeUnitTest
       displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
 
@@ -130,7 +130,7 @@ jobs:
     parameters:
       name: RunMultiNodePythonTests
       displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh pytests
 
@@ -138,6 +138,6 @@ jobs:
     parameters:
       name: RunMultiNodePythonBenchmark
       displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azureuser'
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark

From cb06a648dd56fabe819f5acc42079378f8850c05 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Wed, 5 Aug 2026 13:52:59 -0700
Subject: [PATCH 19/51] remove cuda12

---
 .azure-pipelines/mnnvl-test.yml       | 2 --
 .azure-pipelines/multi-nodes-test.yml | 2 --
 2 files changed, 4 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 4bffbc51a..43e3a5176 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -34,8 +34,6 @@ jobs:
   displayName: Multi nodes test
   strategy:
     matrix:
-      cuda12:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
       cuda13:
         containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 4bffbc51a..43e3a5176 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -34,8 +34,6 @@ jobs:
   displayName: Multi nodes test
   strategy:
     matrix:
-      cuda12:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
       cuda13:
         containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:

From 2ff6758de500c6449f8745cc91c06b9a2fedbc68 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Wed, 5 Aug 2026 16:46:51 -0700
Subject: [PATCH 20/51] aarch64 container image

---
 .azure-pipelines/mnnvl-test.yml       | 2 +-
 .azure-pipelines/multi-nodes-test.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 43e3a5176..bc6cf4d1c 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
   pool:
     name: msccl-ci-gb200
   container:
diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 43e3a5176..bc6cf4d1c 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
   pool:
     name: msccl-ci-gb200
   container:

From e6c53eb609fc98e93bbfd71114cfc7a77303855b Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Wed, 5 Aug 2026 17:26:41 -0700
Subject: [PATCH 21/51] aarch64

---
 .azure-pipelines/mnnvl-test.yml       | 3 ++-
 .azure-pipelines/multi-nodes-test.yml | 3 ++-
 .azure-pipelines/templates/deploy.yml | 6 ++++++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index bc6cf4d1c..5a167c026 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
     name: msccl-ci-gb200
   container:
@@ -106,6 +106,7 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
+      containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
 
   - template: templates/run-remote-task.yml
     parameters:
diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index bc6cf4d1c..5a167c026 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
     name: msccl-ci-gb200
   container:
@@ -106,6 +106,7 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
+      containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
 
   - template: templates/run-remote-task.yml
     parameters:
diff --git a/.azure-pipelines/templates/deploy.yml b/.azure-pipelines/templates/deploy.yml
index 97ecb8374..311347235 100644
--- a/.azure-pipelines/templates/deploy.yml
+++ b/.azure-pipelines/templates/deploy.yml
@@ -41,6 +41,9 @@ parameters:
 - name: sglangImage
   type: string
   default: ''
+- name: containerImage
+  type: string
+  default: ''
 
 steps:
 # 0. Ensure Azure CLI exists before running AzureCLI@2 tasks.
@@ -159,3 +162,6 @@ steps:
     filePath: test/deploy/deploy.sh
     arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} ${{ parameters.pilot }}
     workingDirectory: '$(System.DefaultWorkingDirectory)'
+  ${{ if ne(parameters.containerImage, '') }}:
+    env:
+      CONTAINERIMAGE: ${{ parameters.containerImage }}

From cb1cd6b2ec9b5412a81a046873ab63761f2e7847 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Thu, 6 Aug 2026 10:07:31 -0700
Subject: [PATCH 22/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 5a167c026..07f0b718c 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarm64
   pool:
     name: msccl-ci-gb200
   container:

From 113f2aeaf958a6f7d0e0d55a2568a5b91604324d Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Thu, 6 Aug 2026 10:15:25 -0700
Subject: [PATCH 23/51] test

---
 .azure-pipelines/multi-nodes-test.yml |  2 +-
 test/deploy/run-remote.sh             |  1 +
 test/deploy/setup.sh                  | 25 ++++++++++++-------------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 07f0b718c..5a167c026 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -35,7 +35,7 @@ jobs:
   strategy:
     matrix:
       cuda13:
-        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarm64
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
     name: msccl-ci-gb200
   container:
diff --git a/test/deploy/run-remote.sh b/test/deploy/run-remote.sh
index 2e2f7732a..a86290655 100755
--- a/test/deploy/run-remote.sh
+++ b/test/deploy/run-remote.sh
@@ -113,6 +113,7 @@ PSSH_COMMON=(
 if $USE_DOCKER; then
     INNER="set -euxo pipefail;"
     INNER+=" cd /root/mscclpp;"
+    INNER+=" [ -f /root/mscclpp/.ldpath ] && source /root/mscclpp/.ldpath;"
     INNER+=" export LD_LIBRARY_PATH=/root/mscclpp/build/lib:\\\$LD_LIBRARY_PATH;"
     INNER+=" CMD_B64='${CMD_B64}';"
     INNER+=" TMP=\\\$(mktemp);"
diff --git a/test/deploy/setup.sh b/test/deploy/setup.sh
index 0cb6f4b86..8b672903a 100644
--- a/test/deploy/setup.sh
+++ b/test/deploy/setup.sh
@@ -30,24 +30,23 @@ fi
 
 make -C /root/mscclpp/tools/peer-access-test
 set +e
-/root/mscclpp/tools/peer-access-test/peer_access_test
+# Newer host drivers break with the container's host-injected CUDA compat libs (803);
+# probe without them first, fall back to compat only if the native driver fails.
+NATIVE_LD_PATH=$(echo "${LD_LIBRARY_PATH}" | tr ':' '\n' | grep -v '/compat' | paste -sd ':' -)
+LD_LIBRARY_PATH="${NATIVE_LD_PATH}" /root/mscclpp/tools/peer-access-test/peer_access_test
 PEER_ACCESS_EXIT_CODE=$?
 set -e
-if [ ${PEER_ACCESS_EXIT_CODE} -eq 2 ] && [ "${PLATFORM}" == "cuda" ]; then
-    # Exit code 2 = CUDA init failure (e.g., driver/toolkit version mismatch).
-    # Add CUDA compat libs for forward compatibility and retry.
-    CUDA_COMPAT_PATH="/usr/local/cuda/compat"
-    if [ -d "${CUDA_COMPAT_PATH}" ]; then
-        echo "Adding ${CUDA_COMPAT_PATH} to LD_LIBRARY_PATH for forward compatibility"
-        export LD_LIBRARY_PATH="${CUDA_COMPAT_PATH}:${LD_LIBRARY_PATH}"
-        /root/mscclpp/tools/peer-access-test/peer_access_test
-    else
-        echo "CUDA compat libs not found at ${CUDA_COMPAT_PATH}"
-        exit 1
-    fi
+RESOLVED_LD_PATH="${NATIVE_LD_PATH}"
+if [ ${PEER_ACCESS_EXIT_CODE} -ne 0 ] && [ "${PLATFORM}" == "cuda" ] && [ -d "/usr/local/cuda/compat" ]; then
+    echo "Native driver failed (exit ${PEER_ACCESS_EXIT_CODE}); retrying with CUDA compat libs"
+    RESOLVED_LD_PATH="/usr/local/cuda/compat:${NATIVE_LD_PATH}"
+    LD_LIBRARY_PATH="${RESOLVED_LD_PATH}" /root/mscclpp/tools/peer-access-test/peer_access_test
 elif [ ${PEER_ACCESS_EXIT_CODE} -ne 0 ]; then
     exit ${PEER_ACCESS_EXIT_CODE}
 fi
+# Persist the resolved LD_LIBRARY_PATH so later docker exec sessions (run-remote.sh)
+# reuse it and are not affected by the host runtime's compat-first injection.
+echo "export LD_LIBRARY_PATH=\"${RESOLVED_LD_PATH}\"" > /root/mscclpp/.ldpath
 make -C /root/mscclpp/tools/peer-access-test clean
 
 if [ "${PLATFORM}" == "rocm" ]; then

From 33de0024338142bd7bbd6278c2d3013c4d331119 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Thu, 6 Aug 2026 14:35:50 -0700
Subject: [PATCH 24/51] revert setup.sh changes

---
 test/deploy/setup.sh | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/test/deploy/setup.sh b/test/deploy/setup.sh
index 8b672903a..15e8e179a 100644
--- a/test/deploy/setup.sh
+++ b/test/deploy/setup.sh
@@ -30,23 +30,26 @@ fi
 
 make -C /root/mscclpp/tools/peer-access-test
 set +e
-# Newer host drivers break with the container's host-injected CUDA compat libs (803);
-# probe without them first, fall back to compat only if the native driver fails.
-NATIVE_LD_PATH=$(echo "${LD_LIBRARY_PATH}" | tr ':' '\n' | grep -v '/compat' | paste -sd ':' -)
-LD_LIBRARY_PATH="${NATIVE_LD_PATH}" /root/mscclpp/tools/peer-access-test/peer_access_test
+/root/mscclpp/tools/peer-access-test/peer_access_test
+
 PEER_ACCESS_EXIT_CODE=$?
 set -e
-RESOLVED_LD_PATH="${NATIVE_LD_PATH}"
-if [ ${PEER_ACCESS_EXIT_CODE} -ne 0 ] && [ "${PLATFORM}" == "cuda" ] && [ -d "/usr/local/cuda/compat" ]; then
-    echo "Native driver failed (exit ${PEER_ACCESS_EXIT_CODE}); retrying with CUDA compat libs"
-    RESOLVED_LD_PATH="/usr/local/cuda/compat:${NATIVE_LD_PATH}"
-    LD_LIBRARY_PATH="${RESOLVED_LD_PATH}" /root/mscclpp/tools/peer-access-test/peer_access_test
+if [ ${PEER_ACCESS_EXIT_CODE} -eq 2 ] && [ "${PLATFORM}" == "cuda" ]; then
+    # Exit code 2 = CUDA init failure (e.g., driver/toolkit version mismatch).
+    # Add CUDA compat libs for forward compatibility and retry.
+    CUDA_COMPAT_PATH="/usr/local/cuda/compat"
+    if [ -d "${CUDA_COMPAT_PATH}" ]; then
+        echo "Adding ${CUDA_COMPAT_PATH} to LD_LIBRARY_PATH for forward compatibility"
+        export LD_LIBRARY_PATH="${CUDA_COMPAT_PATH}:${LD_LIBRARY_PATH}"
+        /root/mscclpp/tools/peer-access-test/peer_access_test
+    else
+        echo "CUDA compat libs not found at ${CUDA_COMPAT_PATH}"
+        exit 1
+    fi
 elif [ ${PEER_ACCESS_EXIT_CODE} -ne 0 ]; then
     exit ${PEER_ACCESS_EXIT_CODE}
 fi
-# Persist the resolved LD_LIBRARY_PATH so later docker exec sessions (run-remote.sh)
-# reuse it and are not affected by the host runtime's compat-first injection.
-echo "export LD_LIBRARY_PATH=\"${RESOLVED_LD_PATH}\"" > /root/mscclpp/.ldpath
+
 make -C /root/mscclpp/tools/peer-access-test clean
 
 if [ "${PLATFORM}" == "rocm" ]; then
@@ -75,4 +78,4 @@ pip3 install setuptools_scm
 python3 -m setuptools_scm --force-write-version-files
 
 mkdir -p /var/run/sshd
-/usr/sbin/sshd -p 22345
+/usr/sbin/sshd -p 22345
\ No newline at end of file

From 442fa36a11024d582b0558131d848d131b2b4ea7 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:28:29 -0700
Subject: [PATCH 25/51] remove docker image if it exists already

---
 test/deploy/deploy.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh
index 321f5bb74..ad41060a0 100644
--- a/test/deploy/deploy.sh
+++ b/test/deploy/deploy.sh
@@ -86,6 +86,10 @@ if [ "${PLATFORM}" == "cuda" ]; then
     fi"
 fi
 
+# Remove any leftover container from a previous (possibly failed) run
+parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
+  "sudo docker rm -f ${CONTAINER_NAME} 2>/dev/null || true"
+
 if [ "${CONTAINER_NAME}" == "sglang-mscclpp-test" ]; then
   # force to pull the latest image
   parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \

From c7dfe9315e41030bd70f5a3cc5035115e4ec6506 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:28:49 -0700
Subject: [PATCH 26/51] add gb200 for bench collective

---
 python/mscclpp_benchmark/bench_collective.py | 4 ++--
 python/mscclpp_benchmark/tuning_config.py    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/mscclpp_benchmark/bench_collective.py b/python/mscclpp_benchmark/bench_collective.py
index 0cb11739a..81d371c8e 100644
--- a/python/mscclpp_benchmark/bench_collective.py
+++ b/python/mscclpp_benchmark/bench_collective.py
@@ -199,7 +199,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl
             "default_allreduce_nvls_packet",
             max_message_size=512 * 1024,
             max_nblocks=16,
-            supported_skus=("H100", "GB300"),
+            supported_skus=("H100", "GB300", "GB200"),
             requires_nvls=True,
         ),
         CandidateSpec(
@@ -228,7 +228,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl
             CandidateSpec(
                 "default_allreduce_nvls_zero_copy",
                 max_nblocks=32,
-                supported_skus=("H100", "GB300"),
+                supported_skus=("H100", "GB300", "GB200"),
                 requires_nvls=True,
                 requires_symmetric_memory=True,
             ),
diff --git a/python/mscclpp_benchmark/tuning_config.py b/python/mscclpp_benchmark/tuning_config.py
index 2a914ec95..86e340c0c 100644
--- a/python/mscclpp_benchmark/tuning_config.py
+++ b/python/mscclpp_benchmark/tuning_config.py
@@ -10,7 +10,7 @@
 from pathlib import Path
 from typing import Any
 
-_KNOWN_GPU_SKUS = ("GB300", "MI300X", "H100", "A100")
+_KNOWN_GPU_SKUS = ("GB300", "GB200", "MI300X", "H100", "A100")
 
 
 @dataclass(frozen=True)

From 5f67e4997aa6d42cbd1044e810a2842f052c695a Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:29:13 -0700
Subject: [PATCH 27/51] setup test

---
 .azure-pipelines/mnnvl-test.yml           | 34 ++++-------------------
 .azure-pipelines/templates/deploy.yml     |  6 ----
 .azure-pipelines/templates/mnnvl-test.yml |  0
 3 files changed, 5 insertions(+), 35 deletions(-)
 delete mode 100644 .azure-pipelines/templates/mnnvl-test.yml

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 5a167c026..567470e41 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -106,37 +106,13 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
-      containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
 
   - template: templates/run-remote-task.yml
     parameters:
-      name: RunMscclppTest
-      displayName: Run multi-nodes mscclpp-test
-      continueOnError: true
+      name: PyBench
+      displayName: Run Collective Benchmarks
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodeUnitTest
-      displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodePythonTests
-      displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh pytests
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodePythonBenchmark
-      displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark
+        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include eth0 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x MSCCLPP_SOCKET_IFNAME=eth0 -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
diff --git a/.azure-pipelines/templates/deploy.yml b/.azure-pipelines/templates/deploy.yml
index 311347235..97ecb8374 100644
--- a/.azure-pipelines/templates/deploy.yml
+++ b/.azure-pipelines/templates/deploy.yml
@@ -41,9 +41,6 @@ parameters:
 - name: sglangImage
   type: string
   default: ''
-- name: containerImage
-  type: string
-  default: ''
 
 steps:
 # 0. Ensure Azure CLI exists before running AzureCLI@2 tasks.
@@ -162,6 +159,3 @@ steps:
     filePath: test/deploy/deploy.sh
     arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} ${{ parameters.pilot }}
     workingDirectory: '$(System.DefaultWorkingDirectory)'
-  ${{ if ne(parameters.containerImage, '') }}:
-    env:
-      CONTAINERIMAGE: ${{ parameters.containerImage }}
diff --git a/.azure-pipelines/templates/mnnvl-test.yml b/.azure-pipelines/templates/mnnvl-test.yml
deleted file mode 100644
index e69de29bb..000000000

From 283bc6cbc71f897663180e1b2e2a64b7761b8e4e Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:29:32 -0700
Subject: [PATCH 28/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 34 ++++-----------------------
 1 file changed, 5 insertions(+), 29 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 5a167c026..567470e41 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -106,37 +106,13 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
-      containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0-aarch64
 
   - template: templates/run-remote-task.yml
     parameters:
-      name: RunMscclppTest
-      displayName: Run multi-nodes mscclpp-test
-      continueOnError: true
+      name: PyBench
+      displayName: Run Collective Benchmarks
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodeUnitTest
-      displayName: Run multi-nodes unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodePythonTests
-      displayName: Run multi-nodes python tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh pytests
-
-  - template: templates/run-remote-task.yml
-    parameters:
-      name: RunMultiNodePythonBenchmark
-      displayName: Run multi-nodes python benchmark
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark
+        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include eth0 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x MSCCLPP_SOCKET_IFNAME=eth0 -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune

From 92916a78c86b5ec529851186c86b3e84977c86f7 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:59:17 -0700
Subject: [PATCH 29/51] fix hostfile

---
 .azure-pipelines/mnnvl-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 567470e41..afe8430e7 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -80,8 +80,8 @@ jobs:
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
         # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
         # Bump to more nodes later by parsing further lines the same way.
-        NODE0=$(awk 'NF>=2{print $2; exit}'           <<< "${{ parameters.hostEntries }}")
-        NODE1=$(awk 'NF>=2{c++} c==2{print $2; exit}' <<< "${{ parameters.hostEntries }}")
+        NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
+        NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
 
         echo "Host ${NODE0}
           Port 22345

From 6e388e50beee1d21abdd709b4610f4f472cb7891 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 12:59:23 -0700
Subject: [PATCH 30/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 567470e41..afe8430e7 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -80,8 +80,8 @@ jobs:
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
         # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
         # Bump to more nodes later by parsing further lines the same way.
-        NODE0=$(awk 'NF>=2{print $2; exit}'           <<< "${{ parameters.hostEntries }}")
-        NODE1=$(awk 'NF>=2{c++} c==2{print $2; exit}' <<< "${{ parameters.hostEntries }}")
+        NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
+        NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
 
         echo "Host ${NODE0}
           Port 22345

From bc66427e2c29335864a14f862956414b9bcc9072 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 13:45:43 -0700
Subject: [PATCH 31/51] add python

---
 .azure-pipelines/mnnvl-test.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index afe8430e7..962242fb4 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -78,8 +78,9 @@ jobs:
       script: |
         set -e
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
-        # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
-        # Bump to more nodes later by parsing further lines the same way.
+        # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the
+        # names resolve inside the container too (the DSM hostnames are only in
+        # the agent's /etc/hosts). Bump to more nodes by parsing further lines.
         NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
         NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
 
@@ -113,6 +114,6 @@ jobs:
       displayName: Run Collective Benchmarks
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include eth0 -np 8 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x MSCCLPP_SOCKET_IFNAME=eth0 -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x PYTHONPATH=/root/mscclpp/python -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
           python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune

From fbc6285ef41fc5bc2146110d1f0b454a173b2cc2 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 13:46:16 -0700
Subject: [PATCH 32/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index afe8430e7..962242fb4 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -78,8 +78,9 @@ jobs:
       script: |
         set -e
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
-        # First NVLink pair (2 nodes) from hostEntries; use $2 (the DSM hostname).
-        # Bump to more nodes later by parsing further lines the same way.
+        # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the
+        # names resolve inside the container too (the DSM hostnames are only in
+        # the agent's /etc/hosts). Bump to more nodes by parsing further lines.
         NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
         NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
 
@@ -113,6 +114,6 @@ jobs:
       displayName: Run Collective Benchmarks
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include eth0 -np 8 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x MSCCLPP_SOCKET_IFNAME=eth0 -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x PYTHONPATH=/root/mscclpp/python -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
           python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune

From 8392cb271f5db48ac1fb4ec5f080a461d78e59c8 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Fri, 7 Aug 2026 15:12:01 -0700
Subject: [PATCH 33/51] use venv

---
 .azure-pipelines/mnnvl-test.yml       | 4 ++--
 .azure-pipelines/multi-nodes-test.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 962242fb4..7cbbf5ee7 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -115,5 +115,5 @@ jobs:
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x PYTHONPATH=/root/mscclpp/python -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 962242fb4..7cbbf5ee7 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -115,5 +115,5 @@ jobs:
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
         mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x PYTHONPATH=/root/mscclpp/python -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune

From c4c188b8b46b964e9fb356e1879bdba26d46d622 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Mon, 10 Aug 2026 09:33:43 -0700
Subject: [PATCH 34/51] update build.sh notes

---
 docker/build.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docker/build.sh b/docker/build.sh
index a65d08677..5c22b4f5c 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -96,17 +96,17 @@ echo ""
 echo "    docker tag ${TAG_BASE_DEV} ${GHCR_TAG_BASE_DEV_ARCH} && \\"
 echo "    docker push ${GHCR_TAG_BASE_DEV_ARCH}"
 echo ""
-echo "2. Create or update the multi-arch manifest:"
-echo ""
-echo "   If \`${GHCR_TAG_BASE_DEV}\` already exists (adding another arch):"
+echo "2. (Re)create the multi-arch manifest — ALWAYS list EVERY arch tag you have"
+echo "   published for this target. \`imagetools create\` REPLACES the manifest,"
+echo "   so omitting an arch silently drops it:"
 echo ""
 echo "    docker buildx imagetools create \\"
 echo "        --tag ${GHCR_TAG_BASE_DEV} \\"
-echo "        --append ${GHCR_TAG_BASE_DEV_ARCH}"
+echo "        ${GHCR}:base-dev-${TARGET}-x86_64 \\"
+echo "        ${GHCR}:base-dev-${TARGET}-aarch64"
 echo ""
-echo "   If \`${GHCR_TAG_BASE_DEV}\` does not exist yet:"
+echo "   (Include only the arch tags that actually exist for this target — e.g."
+echo "    ROCm targets are x86_64-only. Verify afterward:)"
 echo ""
-echo "    docker buildx imagetools create \\"
-echo "        --tag ${GHCR_TAG_BASE_DEV} \\"
-echo "        ${GHCR_TAG_BASE_DEV_ARCH}"
+echo "    docker buildx imagetools inspect ${GHCR_TAG_BASE_DEV}"
 echo ""
\ No newline at end of file

From cab622c0e00c59781d6f260dd2702984d0822b1f Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Mon, 10 Aug 2026 09:33:55 -0700
Subject: [PATCH 35/51] imex fix

---
 test/deploy/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh
index ad41060a0..707b05345 100644
--- a/test/deploy/deploy.sh
+++ b/test/deploy/deploy.sh
@@ -103,7 +103,7 @@ else
     "sudo docker pull ${CONTAINERIMAGE}"
 
   # Set GPU passthrough flags based on platform
-  LAUNCH_OPTION="--gpus=all"
+  LAUNCH_OPTION="--gpus=all --device /dev/nvidia-caps-imex-channels/channel0"
   if [ "${PLATFORM}" == "rocm" ]; then
     LAUNCH_OPTION="--device=/dev/kfd --device=/dev/dri --group-add=video"
   fi

From 353d3237e308d9b4c470d028d0faa89ff55bbe19 Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Mon, 10 Aug 2026 19:25:30 +0000
Subject: [PATCH 36/51] apparmor unconfined

---
 test/deploy/deploy.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh
index 707b05345..e1756daf7 100644
--- a/test/deploy/deploy.sh
+++ b/test/deploy/deploy.sh
@@ -108,16 +108,26 @@ else
     LAUNCH_OPTION="--device=/dev/kfd --device=/dev/dri --group-add=video"
   fi
 
+  # The docker-default AppArmor profile triggers a kernel NULL-deref oops in
+  # aa_inet_bind_perm() on the 6.17 azure-nvidia aarch64 kernel used by GB200
+  # nodes. Open MPI's TCP BTL hits it on every bind() during connection setup,
+  # which kills the rank mid-syscall; it is left as a <defunct> zombie and
+  # mpirun then blocks forever on the first cross-node collective. Running the
+  # container unconfined avoids the faulty AppArmor path. This is implied by
+  # --privileged, but is stated explicitly in both branches so the workaround
+  # survives any future change to the privilege flags.
+  SECURITY_OPTION="--security-opt apparmor=unconfined"
+
   if [ "${IB_ENVIRONMENT}" == "true" ]; then
     # InfiniBand: use --privileged for RDMA device access
     parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
-      "sudo docker run --rm -itd --privileged --net=host --ipc=host ${LAUNCH_OPTION} \
+      "sudo docker run --rm -itd --privileged --net=host --ipc=host ${LAUNCH_OPTION} ${SECURITY_OPTION} \
       -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \
       --entrypoint /bin/bash ${CONTAINERIMAGE}"
   else
     # Non-IB: grant SYS_ADMIN and disable seccomp instead of full --privileged
     parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \
-      "sudo docker run --rm -itd --net=host --ipc=host ${LAUNCH_OPTION} --cap-add=SYS_ADMIN --security-opt seccomp=unconfined \
+      "sudo docker run --rm -itd --net=host --ipc=host ${LAUNCH_OPTION} --cap-add=SYS_ADMIN --security-opt seccomp=unconfined ${SECURITY_OPTION} \
       -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \
       --entrypoint /bin/bash ${CONTAINERIMAGE}"
   fi

From a47344f41c97522ca9463d02ade6feceb94656f5 Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Mon, 10 Aug 2026 21:20:42 +0000
Subject: [PATCH 37/51] fix allreduce algo

---
 .azure-pipelines/mnnvl-test.yml                           | 2 +-
 src/ext/collectives/allreduce/allreduce_nvls_zero_copy.cu | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 7cbbf5ee7..7e37810db 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -116,4 +116,4 @@ jobs:
       remoteScript: |
         mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
           -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
+          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory
diff --git a/src/ext/collectives/allreduce/allreduce_nvls_zero_copy.cu b/src/ext/collectives/allreduce/allreduce_nvls_zero_copy.cu
index 48ac67ffc..294ca0ca0 100644
--- a/src/ext/collectives/allreduce/allreduce_nvls_zero_copy.cu
+++ b/src/ext/collectives/allreduce/allreduce_nvls_zero_copy.cu
@@ -179,7 +179,7 @@ std::shared_ptr<void> AllreduceNvls::initAllreduceContext(std::shared_ptr<mscclp
   auto ctx = std::make_shared<AlgorithmCtx>();
   ctx->rank = comm->bootstrap()->getRank();
   ctx->workSize = comm->bootstrap()->getNranks();
-  ctx->nRanksPerNode = comm->bootstrap()->getNranksPerNode();
+  ctx->nRanksPerNode = comm->bootstrap()->getNranksPerIpcDomain();
 
   size_t sendBytes, recvBytes;
   CUdeviceptr sendBasePtr, recvBasePtr;

From 7542da07fce8adc72bb619aff06a28b554bbe098 Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Tue, 11 Aug 2026 00:53:38 +0000
Subject: [PATCH 38/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 7cbbf5ee7..7e37810db 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -116,4 +116,4 @@ jobs:
       remoteScript: |
         mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
           -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --autotune
+          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory

From ca078a7cc306d405114477fb3957e82d77f96b52 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 16:04:55 -0700
Subject: [PATCH 39/51] add mp ut

---
 .azure-pipelines/mnnvl-test.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 7cbbf5ee7..c3dd37321 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -107,6 +107,15 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
+  
+  - template: run-remote-task.yml
+    parameters:
+    name: MpUnitTests
+    displayName: Run mscclpp multi-process unit tests
+    remoteScript: |
+      mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
+      mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
+      mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
 
   - template: templates/run-remote-task.yml
     parameters:

From f2a8ec972e21d3a860a137c6918b0505219d83bb Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 16:05:06 -0700
Subject: [PATCH 40/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 7cbbf5ee7..fca745c00 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -108,6 +108,15 @@ jobs:
       resourceGroup: mscclpp
       gpuArch: '100a'
 
+  - template: run-remote-task.yml
+    parameters:
+    name: MpUnitTests
+    displayName: Run mscclpp multi-process unit tests
+    remoteScript: |
+      mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
+      mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
+      mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
+
   - template: templates/run-remote-task.yml
     parameters:
       name: PyBench

From 45fe34f9b0f76283f34369123fd9bf0ba059c8c6 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 16:08:35 -0700
Subject: [PATCH 41/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 9feef227e..c1585bc7a 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -108,14 +108,14 @@ jobs:
       resourceGroup: mscclpp
       gpuArch: '100a'
 
-  - template: run-remote-task.yml
+  - template: templates/run-remote-task.yml
     parameters:
-    name: MpUnitTests
-    displayName: Run mscclpp multi-process unit tests
-    remoteScript: |
-      mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
-      mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
-      mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
+      name: MpUnitTests
+      displayName: Run mscclpp multi-process unit tests
+      remoteScript: |
+        mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
+        mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
+        mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
 
   - template: templates/run-remote-task.yml
     parameters:

From 304ae9915544e875d3d947c471e2a0ad38514c45 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 16:17:22 -0700
Subject: [PATCH 42/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index c1585bc7a..0d5e0910c 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -112,10 +112,11 @@ jobs:
     parameters:
       name: MpUnitTests
       displayName: Run mscclpp multi-process unit tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
-        mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
-        mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
+        mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests --filter=-Ib
+        mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests --filter=-Ib
+        mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests --filter=-Ib
 
   - template: templates/run-remote-task.yml
     parameters:

From 8ad95fec12e288bbc5bd819cc6322fe29ad360ef Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 17:06:41 -0700
Subject: [PATCH 43/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 0d5e0910c..0dbe9605f 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -107,6 +107,7 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
+      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
 
   - template: templates/run-remote-task.yml
     parameters:
@@ -114,9 +115,13 @@ jobs:
       displayName: Run mscclpp multi-process unit tests
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests --filter=-Ib
-        mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests --filter=-Ib
-        mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests --filter=-Ib
+        for np in 2 4 8; do
+          mpirun --allow-run-as-root -tag-output --bind-to numa \
+            -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
+            -np $np -npernode 4 \
+            -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+            ./build/bin/mp_unit_tests --filter=-Ib
+        done
 
   - template: templates/run-remote-task.yml
     parameters:

From bdb08c8e911f74d61e3ad1b8bea5a4a17c4b5d38 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Tue, 11 Aug 2026 17:34:26 -0700
Subject: [PATCH 44/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 0dbe9605f..4e071c981 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -115,13 +115,11 @@ jobs:
       displayName: Run mscclpp multi-process unit tests
       runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
       remoteScript: |
-        for np in 2 4 8; do
-          mpirun --allow-run-as-root -tag-output --bind-to numa \
-            -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
-            -np $np -npernode 4 \
-            -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-            ./build/bin/mp_unit_tests --filter=-Ib
-        done
+        mpirun --allow-run-as-root -tag-output --bind-to numa \
+          -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
+          -np 1 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --filter=-Ib
 
   - template: templates/run-remote-task.yml
     parameters:

From c006015d55dd5a2b247cb24a62d7c8f502567f92 Mon Sep 17 00:00:00 2001
From: RJ Souza <rjsouza@microsoft.com>
Date: Wed, 12 Aug 2026 09:34:03 -0700
Subject: [PATCH 45/51] working on ut

---
 .azure-pipelines/mnnvl-test.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 5108bb23c..4e071c981 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -107,15 +107,19 @@ jobs:
       pilot: true
       resourceGroup: mscclpp
       gpuArch: '100a'
-  
-  - template: run-remote-task.yml
+      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
+
+  - template: templates/run-remote-task.yml
     parameters:
-    name: MpUnitTests
-    displayName: Run mscclpp multi-process unit tests
-    remoteScript: |
-      mpirun --allow-run-as-root -tag-output -np 2 ./build/bin/mp_unit_tests
-      mpirun --allow-run-as-root -tag-output -np 4 ./build/bin/mp_unit_tests
-      mpirun --allow-run-as-root -tag-output -np 8 ./build/bin/mp_unit_tests
+      name: MpUnitTests
+      displayName: Run mscclpp multi-process unit tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
+      remoteScript: |
+        mpirun --allow-run-as-root -tag-output --bind-to numa \
+          -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
+          -np 1 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --filter=-Ib
 
   - template: templates/run-remote-task.yml
     parameters:

From d4d9c505423fad9560e575ffaaad0f476d5c25ab Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Wed, 12 Aug 2026 17:08:50 +0000
Subject: [PATCH 46/51] add unit tests

---
 .azure-pipelines/mnnvl-test.yml         | 20 +++---
 .azure-pipelines/templates/ut-gb200.yml | 96 +++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 10 deletions(-)
 create mode 100644 .azure-pipelines/templates/ut-gb200.yml

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 4e071c981..3c09343fb 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -109,17 +109,17 @@ jobs:
       gpuArch: '100a'
       cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
 
-  - template: templates/run-remote-task.yml
+  - template: templates/ut-gb200.yml
     parameters:
-      name: MpUnitTests
-      displayName: Run mscclpp multi-process unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        mpirun --allow-run-as-root -tag-output --bind-to numa \
-          -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
-          -np 1 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          ./build/bin/mp_unit_tests --filter=-Ib
+      user: azhpcuser
+      headNode: $(headNode)
+      hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
+      hostfileMpi: /root/mscclpp/test/deploy/hostfile_mpi
+      gpuArch: '100a'
+      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
+      npProcs: 8
+      npPerNode: 4
+      tcpIfInclude: '10.0.5.0/24'
 
   - template: templates/run-remote-task.yml
     parameters:
diff --git a/.azure-pipelines/templates/ut-gb200.yml b/.azure-pipelines/templates/ut-gb200.yml
new file mode 100644
index 000000000..f7f4be50e
--- /dev/null
+++ b/.azure-pipelines/templates/ut-gb200.yml
@@ -0,0 +1,96 @@
+# Unit-test steps for the GB200 / MNNVL (MRC, no-IB) pilot environment.
+#
+# Unlike ut-no-ib-env.yml, this template does NOT deploy or manage a VMSS:
+# mnnvl-test.yml already generates the deploy files, runs deploy.yml with
+# pilot: true, and publishes the head node as $(headNode). This template only
+# runs the unit-test portion on the already-provisioned pilot nodes.
+#
+# Why a dedicated GB200 template is needed: deploy.yml builds mscclpp on the CI
+# agent and deploy.sh ships that build/ tree to the compute nodes. The pilot
+# agent is x86-64 while the GB200 nodes are aarch64, so the shipped
+# ./build/bin/mp_unit_tests (and unit_tests) fail to exec on the nodes with
+# "Exec format error". This template rebuilds the C++ tests natively on every
+# node before running them, so the binaries match the node architecture.
+
+parameters:
+- name: user
+  type: string
+  default: 'azhpcuser'
+# Head node used to launch mpirun (mnnvl-test.yml publishes this as $(headNode)).
+- name: headNode
+  type: string
+# Agent-side hostfile listing every node (user@ip per line). run-remote.sh uses
+# it to build the tests on all nodes in parallel.
+- name: hostfile
+  type: string
+  default: '$(System.DefaultWorkingDirectory)/test/deploy/hostfile'
+# Container-side hostfile passed to mpirun (one ip per line).
+- name: hostfileMpi
+  type: string
+  default: '/root/mscclpp/test/deploy/hostfile_mpi'
+- name: gpuArch
+  type: string
+  default: '100a'
+- name: cmakeArgs
+  type: string
+  default: '-DMSCCLPP_USE_IB=OFF'
+# Total MPI ranks for mp_unit_tests. Must be >= 2 (the Prelim test asserts
+# worldSize >= 2). Default: 4 ranks/node x 2 nodes.
+- name: npProcs
+  type: number
+  default: 8
+- name: npPerNode
+  type: number
+  default: 4
+- name: tcpIfInclude
+  type: string
+  default: '10.0.5.0/24'
+# Negative substring filter for mp_unit_tests (-Ib skips InfiniBand tests).
+- name: filter
+  type: string
+  default: '-Ib'
+# Whether to also run the single-process unit_tests binary.
+- name: runUnitTests
+  type: boolean
+  default: true
+
+steps:
+# Rebuild the C++ tests natively on every node so the binaries are aarch64.
+# The x86-64 CMakeCache.txt shipped from the agent is discarded (rm -rf build).
+- template: run-remote-task.yml
+  parameters:
+    name: BuildGb200Tests
+    displayName: Build unit tests on GB200 nodes
+    runRemoteArgs: '--hostfile ${{ parameters.hostfile }} --user ${{ parameters.user }}'
+    remoteScript: |
+      rm -rf build
+      mkdir -p build && cd build
+      cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DMSCCLPP_BYPASS_GPU_CHECK=ON \
+        -DMSCCLPP_USE_CUDA=ON \
+        -DMSCCLPP_BUILD_TESTS=ON \
+        -DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }} \
+        ${{ parameters.cmakeArgs }} ..
+      make -j unit_tests mp_unit_tests
+
+- ${{ if parameters.runUnitTests }}:
+  - template: run-remote-task.yml
+    parameters:
+      name: Gb200UnitTests
+      displayName: Run mscclpp unit tests
+      runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+      remoteScript: |
+        ./build/bin/unit_tests
+
+- template: run-remote-task.yml
+  parameters:
+    name: Gb200MpUnitTests
+    displayName: Run mscclpp multi-process unit tests
+    runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+    remoteScript: |
+      mpirun --allow-run-as-root -tag-output --bind-to numa \
+        -hostfile ${{ parameters.hostfileMpi }} -mca btl_tcp_if_include ${{ parameters.tcpIfInclude }} \
+        -np ${{ parameters.npProcs }} -npernode ${{ parameters.npPerNode }} \
+        -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+        ./build/bin/mp_unit_tests --filter=${{ parameters.filter }}

From 7fe103cf7283f4b0085e35bf144f2f040bf9ca7a Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Wed, 12 Aug 2026 17:09:12 +0000
Subject: [PATCH 47/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 4e071c981..3c09343fb 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -109,17 +109,17 @@ jobs:
       gpuArch: '100a'
       cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
 
-  - template: templates/run-remote-task.yml
+  - template: templates/ut-gb200.yml
     parameters:
-      name: MpUnitTests
-      displayName: Run mscclpp multi-process unit tests
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
-      remoteScript: |
-        mpirun --allow-run-as-root -tag-output --bind-to numa \
-          -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 \
-          -np 1 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          ./build/bin/mp_unit_tests --filter=-Ib
+      user: azhpcuser
+      headNode: $(headNode)
+      hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
+      hostfileMpi: /root/mscclpp/test/deploy/hostfile_mpi
+      gpuArch: '100a'
+      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
+      npProcs: 8
+      npPerNode: 4
+      tcpIfInclude: '10.0.5.0/24'
 
   - template: templates/run-remote-task.yml
     parameters:

From 0134b99033e57a74af63946db8f5fe9d423effe9 Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Wed, 12 Aug 2026 17:34:48 +0000
Subject: [PATCH 48/51] add port to ut

---
 .azure-pipelines/templates/ut-gb200.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.azure-pipelines/templates/ut-gb200.yml b/.azure-pipelines/templates/ut-gb200.yml
index f7f4be50e..76ff77ab8 100644
--- a/.azure-pipelines/templates/ut-gb200.yml
+++ b/.azure-pipelines/templates/ut-gb200.yml
@@ -49,6 +49,13 @@ parameters:
 - name: filter
   type: string
   default: '-Ib'
+# TcpBootstrap rendezvous port. The bootstrap root address is set to
+# <headNode>:<bootstrapPort> so ranks on other nodes reach rank 0 over the
+# routable subnet. The mp_unit_tests default (127.0.0.1) only works
+# single-node and makes BootstrapTest.*IpPortPair time out across nodes.
+- name: bootstrapPort
+  type: number
+  default: 50053
 # Whether to also run the single-process unit_tests binary.
 - name: runUnitTests
   type: boolean
@@ -93,4 +100,4 @@ steps:
         -hostfile ${{ parameters.hostfileMpi }} -mca btl_tcp_if_include ${{ parameters.tcpIfInclude }} \
         -np ${{ parameters.npProcs }} -npernode ${{ parameters.npPerNode }} \
         -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-        ./build/bin/mp_unit_tests --filter=${{ parameters.filter }}
+        ./build/bin/mp_unit_tests --ip_port=${{ parameters.headNode }}:${{ parameters.bootstrapPort }} --filter=${{ parameters.filter }}

From d89b7dbea1989164fd10d74077755d16c4886046 Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Thu, 13 Aug 2026 02:21:43 +0000
Subject: [PATCH 49/51] work on UT

---
 .azure-pipelines/mnnvl-test.yml         |  5 --
 .azure-pipelines/templates/ut-gb200.yml | 89 ++++++++++++++-----------
 2 files changed, 51 insertions(+), 43 deletions(-)

diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml
index 3c09343fb..0e7a95746 100644
--- a/.azure-pipelines/mnnvl-test.yml
+++ b/.azure-pipelines/mnnvl-test.yml
@@ -114,12 +114,7 @@ jobs:
       user: azhpcuser
       headNode: $(headNode)
       hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
-      hostfileMpi: /root/mscclpp/test/deploy/hostfile_mpi
       gpuArch: '100a'
-      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
-      npProcs: 8
-      npPerNode: 4
-      tcpIfInclude: '10.0.5.0/24'
 
   - template: templates/run-remote-task.yml
     parameters:
diff --git a/.azure-pipelines/templates/ut-gb200.yml b/.azure-pipelines/templates/ut-gb200.yml
index 76ff77ab8..ce9595c90 100644
--- a/.azure-pipelines/templates/ut-gb200.yml
+++ b/.azure-pipelines/templates/ut-gb200.yml
@@ -1,17 +1,3 @@
-# Unit-test steps for the GB200 / MNNVL (MRC, no-IB) pilot environment.
-#
-# Unlike ut-no-ib-env.yml, this template does NOT deploy or manage a VMSS:
-# mnnvl-test.yml already generates the deploy files, runs deploy.yml with
-# pilot: true, and publishes the head node as $(headNode). This template only
-# runs the unit-test portion on the already-provisioned pilot nodes.
-#
-# Why a dedicated GB200 template is needed: deploy.yml builds mscclpp on the CI
-# agent and deploy.sh ships that build/ tree to the compute nodes. The pilot
-# agent is x86-64 while the GB200 nodes are aarch64, so the shipped
-# ./build/bin/mp_unit_tests (and unit_tests) fail to exec on the nodes with
-# "Exec format error". This template rebuilds the C++ tests natively on every
-# node before running them, so the binaries match the node architecture.
-
 parameters:
 - name: user
   type: string
@@ -25,30 +11,37 @@ parameters:
   type: string
   default: '$(System.DefaultWorkingDirectory)/test/deploy/hostfile'
 # Container-side hostfile passed to mpirun (one ip per line).
-- name: hostfileMpi
-  type: string
-  default: '/root/mscclpp/test/deploy/hostfile_mpi'
 - name: gpuArch
   type: string
   default: '100a'
-- name: cmakeArgs
+# Single-node rank ladder for mp_unit_tests. Space-separated; each value is one
+# mpirun -np run on the head node. GB200 nodes have 4 GPUs, so 2 and 4 exercise
+# the intra-node paths (IPC / NVLink). Every value must be >= 2 (the Prelim test
+# asserts worldSize >= 2). The communicator/channel/executor tests only have a
+# working transport intra-node in a no-IB build, so they must run single-node.
+- name: singleNodeNps
   type: string
-  default: '-DMSCCLPP_USE_IB=OFF'
-# Total MPI ranks for mp_unit_tests. Must be >= 2 (the Prelim test asserts
-# worldSize >= 2). Default: 4 ranks/node x 2 nodes.
-- name: npProcs
-  type: number
-  default: 8
-- name: npPerNode
-  type: number
-  default: 4
-- name: tcpIfInclude
-  type: string
-  default: '10.0.5.0/24'
-# Negative substring filter for mp_unit_tests (-Ib skips InfiniBand tests).
+  default: '2 4'
+# Negative substring filter for the single-node runs. Comma-separated patterns
+# are all excluded. -Ib skips the InfiniBand tests (no IB on MNNVL).
 - name: filter
   type: string
   default: '-Ib'
+# Whether to also run a cross-node mp_unit_tests pass. It validates the
+# multi-node TcpBootstrap rendezvous over the routable subnet -- the path unique
+# to the MNNVL topology -- using only the tests that work cross-node without IB.
+- name: runCrossNodeBootstrap
+  type: boolean
+  default: true
+# Negative filter for the cross-node pass. Excludes every fixture that needs a
+# cross-node data transport (Communicator / Executor / *Channel / Semaphore /
+# Ib), which a no-IB build lacks between nodes and which otherwise deadlocks on
+# the mesh barrier. Also excludes TimeoutWithId, whose 100ms window on a 1s
+# timeout is too tight for remote ranks over TCP. What remains: BootstrapTest.*
+# and MultiProcessTest.* (Prelim / HostName / HostHash).
+- name: crossNodeFilter
+  type: string
+  default: '-Communicator,Executor,Ib,MemoryChannel,PortChannel,SemaphorePerf,SwitchChannel,TimeoutWithId'
 # TcpBootstrap rendezvous port. The bootstrap root address is set to
 # <headNode>:<bootstrapPort> so ranks on other nodes reach rank 0 over the
 # routable subnet. The mp_unit_tests default (127.0.0.1) only works
@@ -78,7 +71,7 @@ steps:
         -DMSCCLPP_USE_CUDA=ON \
         -DMSCCLPP_BUILD_TESTS=ON \
         -DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }} \
-        ${{ parameters.cmakeArgs }} ..
+        -DMSCCLPP_USE_IB=OFF ..
       make -j unit_tests mp_unit_tests
 
 - ${{ if parameters.runUnitTests }}:
@@ -90,14 +83,34 @@ steps:
       remoteScript: |
         ./build/bin/unit_tests
 
+# Run mp_unit_tests single-node on the head node across the rank ladder. No
+# hostfile / ip_port needed: all ranks are local, so loopback bootstrap works and
+# every fixture has a working intra-node transport (IPC / NVLink).
 - template: run-remote-task.yml
   parameters:
     name: Gb200MpUnitTests
-    displayName: Run mscclpp multi-process unit tests
+    displayName: Run mscclpp multi-process unit tests (single-node)
     runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
     remoteScript: |
-      mpirun --allow-run-as-root -tag-output --bind-to numa \
-        -hostfile ${{ parameters.hostfileMpi }} -mca btl_tcp_if_include ${{ parameters.tcpIfInclude }} \
-        -np ${{ parameters.npProcs }} -npernode ${{ parameters.npPerNode }} \
-        -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-        ./build/bin/mp_unit_tests --ip_port=${{ parameters.headNode }}:${{ parameters.bootstrapPort }} --filter=${{ parameters.filter }}
+      for np in ${{ parameters.singleNodeNps }}; do
+        echo "=== mp_unit_tests -np ${np} (single-node) ==="
+        mpirun --allow-run-as-root -tag-output --bind-to numa -np ${np} \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --filter=${{ parameters.filter }}
+      done
+
+# Cross-node pass: validate the multi-node TcpBootstrap rendezvous. mpirun spans
+# both nodes via the container hostfile; --ip_port points every rank at rank 0 on
+# the head node (the mp_unit_tests default of 127.0.0.1 only works single-node
+# and makes the *IpPortPair tests time out across nodes). crossNodeFilter keeps
+# this to the fixtures that work without cross-node IB.
+- ${{ if parameters.runCrossNodeBootstrap }}:
+  - template: run-remote-task.yml
+    parameters:
+      name: Gb200MpBootstrapMultiNode
+      displayName: Run mscclpp multi-process bootstrap tests (cross-node)
+      runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}'
+      remoteScript: |
+        mpirun --allow-run-as-root -tag-output --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
+          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
+          ./build/bin/mp_unit_tests --ip_port=${{ parameters.headNode }}:${{ parameters.bootstrapPort }} --filter=${{ parameters.crossNodeFilter }}

From 9be4c2f743239e2e449432cb5a099d8b0ef75dec Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Thu, 13 Aug 2026 02:22:09 +0000
Subject: [PATCH 50/51] test

---
 .azure-pipelines/multi-nodes-test.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 3c09343fb..0e7a95746 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -114,12 +114,7 @@ jobs:
       user: azhpcuser
       headNode: $(headNode)
       hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
-      hostfileMpi: /root/mscclpp/test/deploy/hostfile_mpi
       gpuArch: '100a'
-      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
-      npProcs: 8
-      npPerNode: 4
-      tcpIfInclude: '10.0.5.0/24'
 
   - template: templates/run-remote-task.yml
     parameters:

From 42b6c561013383c4b10f677486b0379b4907970a Mon Sep 17 00:00:00 2001
From: Empyreus <rjsouza1995@gmail.com>
Date: Thu, 13 Aug 2026 03:59:00 +0000
Subject: [PATCH 51/51] revert multinode

---
 .azure-pipelines/multi-nodes-test.yml | 90 +++++++++++++--------------
 1 file changed, 44 insertions(+), 46 deletions(-)

diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml
index 0e7a95746..730825a4a 100644
--- a/.azure-pipelines/multi-nodes-test.yml
+++ b/.azure-pipelines/multi-nodes-test.yml
@@ -17,45 +17,28 @@ pr: none
 parameters:
 - name: vmssName
   type: string
-  default: ''   # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
+  default: mscclpp-h100-multinode-ci
 - name: hostEntries
   type: string
   default: |
-    10.0.5.145 DSM121082302011
-    10.0.5.150 DSM121082302014
-    10.0.5.177 DSM121082301013
-    10.0.5.225 DSM121082301032
-
-variables:
-  - group: mscclpp
+    10.0.0.5 mscclpp-h100-multinode-ci000000
+    10.0.0.4 mscclpp-h100-multinode-ci000001
 
 jobs:
 - job: MultiNodesTest
   displayName: Multi nodes test
   strategy:
     matrix:
+      cuda12:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
       cuda13:
         containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
   pool:
-    name: msccl-ci-gb200
+    name: mscclpp-multi-node
   container:
     image: $[ variables['containerImage'] ]
 
   steps:
-  - task: Bash@3
-    displayName: Node connectivity check
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -x
-        echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
-        while IFS= read -r line; do
-          ip=$(awk 'NF>=1{print $1}' <<< "$line")
-          [ -z "$ip" ] && continue
-          timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
-            && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
-        done <<< "${{ parameters.hostEntries }}"
-
   - task: Bash@3
     displayName: Add HostEntry
     inputs:
@@ -77,12 +60,10 @@ jobs:
       targetType: 'inline'
       script: |
         set -e
+        VMSS="${{ parameters.vmssName }}"
         DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
-        # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the
-        # names resolve inside the container too (the DSM hostnames are only in
-        # the agent's /etc/hosts). Bump to more nodes by parsing further lines.
-        NODE0=$(awk 'NF>=2{print $1; exit}'           <<< "${{ parameters.hostEntries }}")
-        NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
+        NODE0="${VMSS}000000"
+        NODE1="${VMSS}000001"
 
         echo "Host ${NODE0}
           Port 22345
@@ -93,35 +74,52 @@ jobs:
           IdentityFile /root/mscclpp/sshkey
           StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
 
-        printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+        printf '%s\n%s\n' "azureuser@${NODE0}" "azureuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
 
         printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
 
-        # Publish the head node so the run steps can target it without hardcoding.
-        echo "##vso[task.setvariable variable=headNode]${NODE0}"
-
   - template: templates/deploy.yml
     parameters:
       subscription: mscclpp-ci-h100
       vmssName: ${{ parameters.vmssName }}
-      pilot: true
       resourceGroup: mscclpp
-      gpuArch: '100a'
-      cmakeArgs: '-DMSCCLPP_USE_IB=OFF'
+      gpuArch: '90'
 
-  - template: templates/ut-gb200.yml
+  - template: templates/run-remote-task.yml
     parameters:
-      user: azhpcuser
-      headNode: $(headNode)
-      hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile
-      gpuArch: '100a'
+      name: RunMscclppTest
+      displayName: Run multi-nodes mscclpp-test
+      continueOnError: true
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh mscclpp-test
 
   - template: templates/run-remote-task.yml
     parameters:
-      name: PyBench
-      displayName: Run Collective Benchmarks
-      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
+      name: RunMultiNodeUnitTest
+      displayName: Run multi-nodes unit tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
       remoteScript: |
-        mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \
-          -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
-          /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory
+        bash /root/mscclpp/test/deploy/run_tests.sh mp-ut
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMultiNodePythonTests
+      displayName: Run multi-nodes python tests
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh pytests
+
+  - template: templates/run-remote-task.yml
+    parameters:
+      name: RunMultiNodePythonBenchmark
+      displayName: Run multi-nodes python benchmark
+      runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host ${{ parameters.vmssName }}000000 --user azureuser'
+      remoteScript: |
+        bash /root/mscclpp/test/deploy/run_tests.sh py-benchmark
+
+  - template: templates/stop.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+      resourceGroup: mscclpp
\ No newline at end of file