diff --git a/.azure-pipelines/mnnvl-test.yml b/.azure-pipelines/mnnvl-test.yml new file mode 100644 index 000000000..0e7a95746 --- /dev/null +++ b/.azure-pipelines/mnnvl-test.yml @@ -0,0 +1,127 @@ +trigger: + branches: + include: + - main + - release/* + paths: + exclude: + - .devcontainer/** + - .github/** + - docker/** + - docs/** + - '**/*.md' + +# Do not run multi-nodes-test for PR, we can trigger it manually +pr: none + +parameters: +- name: vmssName + type: string + default: '' # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml +- name: hostEntries + type: string + default: | + 10.0.5.145 DSM121082302011 + 10.0.5.150 DSM121082302014 + 10.0.5.177 DSM121082301013 + 10.0.5.225 DSM121082301032 + +variables: + - group: mscclpp + +jobs: +- job: MultiNodesTest + displayName: Multi nodes test + strategy: + matrix: + cuda13: + containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0 + pool: + name: msccl-ci-gb200 + container: + image: $[ variables['containerImage'] ] + + steps: + - task: Bash@3 + displayName: Node connectivity check + inputs: + targetType: 'inline' + script: | + set -x + echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo + while IFS= read -r line; do + ip=$(awk 'NF>=1{print $1}' <<< "$line") + [ -z "$ip" ] && continue + timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \ + && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE" + done <<< "${{ parameters.hostEntries }}" + + - task: Bash@3 + displayName: Add HostEntry + inputs: + targetType: 'inline' + script: | + while IFS= read -r line; do + [ -z "$line" ] && continue + if ! grep -qxF "$line" /etc/hosts; then + echo "Adding to /etc/hosts: $line" + echo "$line" | sudo tee -a /etc/hosts + else + echo "Entry already exists: $line" + fi + done <<< "${{ parameters.hostEntries }}" + + - task: Bash@3 + displayName: Generate deploy files + inputs: + targetType: 'inline' + script: | + set -e + DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy" + # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the + # names resolve inside the container too (the DSM hostnames are only in + # the agent's /etc/hosts). Bump to more nodes by parsing further lines. + NODE0=$(awk 'NF>=2{print $1; exit}' <<< "${{ parameters.hostEntries }}") + NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}") + + echo "Host ${NODE0} + Port 22345 + IdentityFile /root/mscclpp/sshkey + StrictHostKeyChecking no + Host ${NODE1} + Port 22345 + IdentityFile /root/mscclpp/sshkey + StrictHostKeyChecking no" > "${DEPLOY_DIR}/config" + + printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile" + + printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi" + + # Publish the head node so the run steps can target it without hardcoding. + echo "##vso[task.setvariable variable=headNode]${NODE0}" + + - template: templates/deploy.yml + parameters: + subscription: mscclpp-ci-h100 + vmssName: ${{ parameters.vmssName }} + pilot: true + resourceGroup: mscclpp + gpuArch: '100a' + cmakeArgs: '-DMSCCLPP_USE_IB=OFF' + + - template: templates/ut-gb200.yml + parameters: + user: azhpcuser + headNode: $(headNode) + hostfile: $(System.DefaultWorkingDirectory)/test/deploy/hostfile + gpuArch: '100a' + + - template: templates/run-remote-task.yml + parameters: + name: PyBench + displayName: Run Collective Benchmarks + runRemoteArgs: '--hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser' + remoteScript: | + mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \ + -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \ + /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory diff --git a/.azure-pipelines/multi-nodes-test.yml b/.azure-pipelines/multi-nodes-test.yml index c4e27be47..730825a4a 100644 --- a/.azure-pipelines/multi-nodes-test.yml +++ b/.azure-pipelines/multi-nodes-test.yml @@ -122,4 +122,4 @@ jobs: parameters: subscription: mscclpp-ci-h100 vmssName: ${{ parameters.vmssName }} - resourceGroup: mscclpp + resourceGroup: mscclpp \ No newline at end of file diff --git a/.azure-pipelines/templates/deploy.yml b/.azure-pipelines/templates/deploy.yml index 9d9e9abf7..cbe5c84ba 100644 --- a/.azure-pipelines/templates/deploy.yml +++ b/.azure-pipelines/templates/deploy.yml @@ -6,6 +6,9 @@ parameters: - name: resourceGroup type: string default: mscclpp +- name: pilot + type: boolean + default: false # Build parameters - name: platform type: string @@ -136,15 +139,16 @@ steps: sudo apt-get update -y sudo apt-get install pssh -y -- task: AzureCLI@2 - name: StartVMSS - displayName: Start VMSS - inputs: - azureSubscription: ${{ parameters.subscription }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }} +- ${{ if not(parameters.pilot) }}: + - task: AzureCLI@2 + name: StartVMSS + displayName: Start VMSS + inputs: + azureSubscription: ${{ parameters.subscription }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }} # 4. Deploy test environment - task: Bash@3 @@ -153,5 +157,5 @@ steps: inputs: targetType: filePath filePath: test/deploy/deploy.sh - arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} + arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }} ${{ parameters.sglangImage }} ${{ parameters.pilot }} workingDirectory: '$(System.DefaultWorkingDirectory)' diff --git a/.azure-pipelines/templates/ut-gb200.yml b/.azure-pipelines/templates/ut-gb200.yml new file mode 100644 index 000000000..ce9595c90 --- /dev/null +++ b/.azure-pipelines/templates/ut-gb200.yml @@ -0,0 +1,116 @@ +parameters: +- name: user + type: string + default: 'azhpcuser' +# Head node used to launch mpirun (mnnvl-test.yml publishes this as $(headNode)). +- name: headNode + type: string +# Agent-side hostfile listing every node (user@ip per line). run-remote.sh uses +# it to build the tests on all nodes in parallel. +- name: hostfile + type: string + default: '$(System.DefaultWorkingDirectory)/test/deploy/hostfile' +# Container-side hostfile passed to mpirun (one ip per line). +- name: gpuArch + type: string + default: '100a' +# Single-node rank ladder for mp_unit_tests. Space-separated; each value is one +# mpirun -np run on the head node. GB200 nodes have 4 GPUs, so 2 and 4 exercise +# the intra-node paths (IPC / NVLink). Every value must be >= 2 (the Prelim test +# asserts worldSize >= 2). The communicator/channel/executor tests only have a +# working transport intra-node in a no-IB build, so they must run single-node. +- name: singleNodeNps + type: string + default: '2 4' +# Negative substring filter for the single-node runs. Comma-separated patterns +# are all excluded. -Ib skips the InfiniBand tests (no IB on MNNVL). +- name: filter + type: string + default: '-Ib' +# Whether to also run a cross-node mp_unit_tests pass. It validates the +# multi-node TcpBootstrap rendezvous over the routable subnet -- the path unique +# to the MNNVL topology -- using only the tests that work cross-node without IB. +- name: runCrossNodeBootstrap + type: boolean + default: true +# Negative filter for the cross-node pass. Excludes every fixture that needs a +# cross-node data transport (Communicator / Executor / *Channel / Semaphore / +# Ib), which a no-IB build lacks between nodes and which otherwise deadlocks on +# the mesh barrier. Also excludes TimeoutWithId, whose 100ms window on a 1s +# timeout is too tight for remote ranks over TCP. What remains: BootstrapTest.* +# and MultiProcessTest.* (Prelim / HostName / HostHash). +- name: crossNodeFilter + type: string + default: '-Communicator,Executor,Ib,MemoryChannel,PortChannel,SemaphorePerf,SwitchChannel,TimeoutWithId' +# TcpBootstrap rendezvous port. The bootstrap root address is set to +# : so ranks on other nodes reach rank 0 over the +# routable subnet. The mp_unit_tests default (127.0.0.1) only works +# single-node and makes BootstrapTest.*IpPortPair time out across nodes. +- name: bootstrapPort + type: number + default: 50053 +# Whether to also run the single-process unit_tests binary. +- name: runUnitTests + type: boolean + default: true + +steps: +# Rebuild the C++ tests natively on every node so the binaries are aarch64. +# The x86-64 CMakeCache.txt shipped from the agent is discarded (rm -rf build). +- template: run-remote-task.yml + parameters: + name: BuildGb200Tests + displayName: Build unit tests on GB200 nodes + runRemoteArgs: '--hostfile ${{ parameters.hostfile }} --user ${{ parameters.user }}' + remoteScript: | + rm -rf build + mkdir -p build && cd build + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DMSCCLPP_BYPASS_GPU_CHECK=ON \ + -DMSCCLPP_USE_CUDA=ON \ + -DMSCCLPP_BUILD_TESTS=ON \ + -DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }} \ + -DMSCCLPP_USE_IB=OFF .. + make -j unit_tests mp_unit_tests + +- ${{ if parameters.runUnitTests }}: + - template: run-remote-task.yml + parameters: + name: Gb200UnitTests + displayName: Run mscclpp unit tests + runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}' + remoteScript: | + ./build/bin/unit_tests + +# Run mp_unit_tests single-node on the head node across the rank ladder. No +# hostfile / ip_port needed: all ranks are local, so loopback bootstrap works and +# every fixture has a working intra-node transport (IPC / NVLink). +- template: run-remote-task.yml + parameters: + name: Gb200MpUnitTests + displayName: Run mscclpp multi-process unit tests (single-node) + runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}' + remoteScript: | + for np in ${{ parameters.singleNodeNps }}; do + echo "=== mp_unit_tests -np ${np} (single-node) ===" + mpirun --allow-run-as-root -tag-output --bind-to numa -np ${np} \ + -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \ + ./build/bin/mp_unit_tests --filter=${{ parameters.filter }} + done + +# Cross-node pass: validate the multi-node TcpBootstrap rendezvous. mpirun spans +# both nodes via the container hostfile; --ip_port points every rank at rank 0 on +# the head node (the mp_unit_tests default of 127.0.0.1 only works single-node +# and makes the *IpPortPair tests time out across nodes). crossNodeFilter keeps +# this to the fixtures that work without cross-node IB. +- ${{ if parameters.runCrossNodeBootstrap }}: + - template: run-remote-task.yml + parameters: + name: Gb200MpBootstrapMultiNode + displayName: Run mscclpp multi-process bootstrap tests (cross-node) + runRemoteArgs: '--host ${{ parameters.headNode }} --user ${{ parameters.user }}' + remoteScript: | + mpirun --allow-run-as-root -tag-output --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include 10.0.5.0/24 -np 8 -npernode 4 \ + -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \ + ./build/bin/mp_unit_tests --ip_port=${{ parameters.headNode }}:${{ parameters.bootstrapPort }} --filter=${{ parameters.crossNodeFilter }} diff --git a/docker/build.sh b/docker/build.sh index a65d08677..5c22b4f5c 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -96,17 +96,17 @@ echo "" echo " docker tag ${TAG_BASE_DEV} ${GHCR_TAG_BASE_DEV_ARCH} && \\" echo " docker push ${GHCR_TAG_BASE_DEV_ARCH}" echo "" -echo "2. Create or update the multi-arch manifest:" -echo "" -echo " If \`${GHCR_TAG_BASE_DEV}\` already exists (adding another arch):" +echo "2. (Re)create the multi-arch manifest — ALWAYS list EVERY arch tag you have" +echo " published for this target. \`imagetools create\` REPLACES the manifest," +echo " so omitting an arch silently drops it:" echo "" echo " docker buildx imagetools create \\" echo " --tag ${GHCR_TAG_BASE_DEV} \\" -echo " --append ${GHCR_TAG_BASE_DEV_ARCH}" +echo " ${GHCR}:base-dev-${TARGET}-x86_64 \\" +echo " ${GHCR}:base-dev-${TARGET}-aarch64" echo "" -echo " If \`${GHCR_TAG_BASE_DEV}\` does not exist yet:" +echo " (Include only the arch tags that actually exist for this target — e.g." +echo " ROCm targets are x86_64-only. Verify afterward:)" echo "" -echo " docker buildx imagetools create \\" -echo " --tag ${GHCR_TAG_BASE_DEV} \\" -echo " ${GHCR_TAG_BASE_DEV_ARCH}" +echo " docker buildx imagetools inspect ${GHCR_TAG_BASE_DEV}" echo "" \ No newline at end of file diff --git a/python/mscclpp_benchmark/bench_collective.py b/python/mscclpp_benchmark/bench_collective.py index 35f30914a..5d8cba0b5 100644 --- a/python/mscclpp_benchmark/bench_collective.py +++ b/python/mscclpp_benchmark/bench_collective.py @@ -199,6 +199,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl "default_allreduce_nvls_packet", max_message_size=512 * 1024, max_nblocks=16, + supported_skus=("H100", "GB300", "GB200"), requires_nvls=True, ), CandidateSpec( @@ -231,6 +232,7 @@ def _candidate_specs(collective: str, *, symmetric_memory: bool = False) -> tupl CandidateSpec( "default_allreduce_nvls_zero_copy", max_nblocks=32, + supported_skus=("H100", "GB300", "GB200"), requires_nvls=True, requires_symmetric_memory=True, ), diff --git a/python/mscclpp_benchmark/tuning_config.py b/python/mscclpp_benchmark/tuning_config.py index 2a914ec95..86e340c0c 100644 --- a/python/mscclpp_benchmark/tuning_config.py +++ b/python/mscclpp_benchmark/tuning_config.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any -_KNOWN_GPU_SKUS = ("GB300", "MI300X", "H100", "A100") +_KNOWN_GPU_SKUS = ("GB300", "GB200", "MI300X", "H100", "A100") @dataclass(frozen=True) diff --git a/test/deploy/deploy.sh b/test/deploy/deploy.sh index e237bd3a1..5e5f2db2b 100644 --- a/test/deploy/deploy.sh +++ b/test/deploy/deploy.sh @@ -18,6 +18,7 @@ IB_ENVIRONMENT="${2:-true}" PLATFORM="${3:-cuda}" CONTAINER_NAME="${4:-mscclpp-test}" SGLANG_IMAGE="${5:-lmsysorg/sglang:latest}" +PILOT="$(echo "${6:-false}" | tr '[:upper:]' '[:lower:]')" KeyFilePath=${SSHKEYFILE_SECUREFILEPATH} ROOT_DIR="${SYSTEM_DEFAULTWORKINGDIRECTORY}/" @@ -31,12 +32,23 @@ fi SSH_OPTION="StrictHostKeyChecking=no" +# CI-agent -> node SSH args. Optionally route through a jumpbox via ProxyCommand +# (opt-in through SSH_PROXY_JUMP; SSH_PROXY_KEY selects a separate jump key). +# When the vars are unset there is no proxy, so pipelines that don't set them +# (e.g. multi-nodes-test.yml) keep using direct SSH unchanged. +SSH_EXTRA_ARGS="-i ${KeyFilePath}" +if [ -n "${SSH_PROXY_JUMP:-}" ]; then + PROXY_KEY_OPT="" + [ -n "${SSH_PROXY_KEY:-}" ] && PROXY_KEY_OPT="-i ${SSH_PROXY_KEY} " + SSH_EXTRA_ARGS="${SSH_EXTRA_ARGS} -o ProxyCommand=\"ssh ${PROXY_KEY_OPT}-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p ${SSH_PROXY_JUMP}\"" +fi + chmod 400 ${KeyFilePath} ssh-keygen -t rsa -f sshkey -P "" while true; do set +e - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "hostname" + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "hostname" if [ $? -eq 0 ]; then break fi @@ -45,21 +57,21 @@ while true; do done set -e -parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "sudo rm -rf ${DST_DIR}" +parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "sudo rm -rf ${DST_DIR}" tar czf /tmp/mscclpp.tar.gz -C ${ROOT_DIR} . -parallel-scp -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION /tmp/mscclpp.tar.gz /tmp/mscclpp.tar.gz -parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ +parallel-scp -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION /tmp/mscclpp.tar.gz /tmp/mscclpp.tar.gz +parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "sudo mkdir -p ${DST_DIR} && sudo tar xzf /tmp/mscclpp.tar.gz -C ${DST_DIR} && sudo rm -f /tmp/mscclpp.tar.gz" rm -f /tmp/mscclpp.tar.gz if [ "${PLATFORM}" == "rocm" ]; then - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION "sudo modprobe amdgpu" + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION "sudo modprobe amdgpu" fi # Install GDRCopy kernel module on host VMs (CUDA only) GDRCOPY_VERSION="2.5.2" if [ "${PLATFORM}" == "cuda" ]; then - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "if lsmod | grep -q gdrdrv; then echo 'gdrdrv module already loaded' else @@ -81,36 +93,46 @@ parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ if [ "${CONTAINER_NAME}" == "sglang-mscclpp-test" ]; then # force to pull the latest image - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "sudo docker pull ${SGLANG_IMAGE}" - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "sudo docker run --rm -itd --name=${CONTAINER_NAME} --privileged --net=host --ipc=host --gpus=all -w /root -v ${DST_DIR}:/root/mscclpp --entrypoint /bin/bash ${SGLANG_IMAGE}" else # force to pull the latest image - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "sudo docker pull ${CONTAINERIMAGE}" # Set GPU passthrough flags based on platform - LAUNCH_OPTION="--gpus=all" + LAUNCH_OPTION="--gpus=all --device /dev/nvidia-caps-imex-channels/channel0" if [ "${PLATFORM}" == "rocm" ]; then LAUNCH_OPTION="--device=/dev/kfd --device=/dev/dri --group-add=video" fi + # The docker-default AppArmor profile triggers a kernel NULL-deref oops in + # aa_inet_bind_perm() on the 6.17 azure-nvidia aarch64 kernel used by GB200 + # nodes. Open MPI's TCP BTL hits it on every bind() during connection setup, + # which kills the rank mid-syscall; it is left as a zombie and + # mpirun then blocks forever on the first cross-node collective. Running the + # container unconfined avoids the faulty AppArmor path. This is implied by + # --privileged, but is stated explicitly in both branches so the workaround + # survives any future change to the privilege flags. + SECURITY_OPTION="--security-opt apparmor=unconfined" + if [ "${IB_ENVIRONMENT}" == "true" ]; then # InfiniBand: use --privileged for RDMA device access - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ - "sudo docker run --rm -itd --privileged --net=host --ipc=host ${LAUNCH_OPTION} \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ + "sudo docker run --rm -itd --privileged --net=host --ipc=host ${LAUNCH_OPTION} ${SECURITY_OPTION} \ -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \ --entrypoint /bin/bash ${CONTAINERIMAGE}" else # Non-IB: grant SYS_ADMIN and disable seccomp instead of full --privileged - parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ - "sudo docker run --rm -itd --net=host --ipc=host ${LAUNCH_OPTION} --cap-add=SYS_ADMIN --security-opt seccomp=unconfined \ + parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ + "sudo docker run --rm -itd --net=host --ipc=host ${LAUNCH_OPTION} --cap-add=SYS_ADMIN --security-opt seccomp=unconfined ${SECURITY_OPTION} \ -w /root -v ${DST_DIR}:/root/mscclpp -v /opt/microsoft:/opt/microsoft --ulimit memlock=-1:-1 --name=${CONTAINER_NAME} \ --entrypoint /bin/bash ${CONTAINERIMAGE}" fi fi -parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION \ +parallel-ssh -i -t 0 -h ${HOSTFILE} -x "${SSH_EXTRA_ARGS}" -O $SSH_OPTION \ "sudo docker exec -t --user root ${CONTAINER_NAME} bash '/root/mscclpp/test/deploy/setup.sh' ${PLATFORM}" diff --git a/test/deploy/run-remote.sh b/test/deploy/run-remote.sh index 9607664fc..a86290655 100755 --- a/test/deploy/run-remote.sh +++ b/test/deploy/run-remote.sh @@ -20,6 +20,17 @@ HOSTFILE="${SCRIPT_DIR}/hostfile_ci" SSH_OPTION="StrictHostKeyChecking=no" KeyFilePath="${SSHKEYFILE_SECUREFILEPATH}" +# CI-agent -> node SSH args. Optionally route through a jumpbox via ProxyCommand +# (opt-in through SSH_PROXY_JUMP; SSH_PROXY_KEY selects a separate jump key). +# When the vars are unset there is no proxy, so pipelines that don't set them +# keep using direct SSH unchanged. +SSH_EXTRA_ARGS="-i ${KeyFilePath}" +if [ -n "${SSH_PROXY_JUMP:-}" ]; then + PROXY_KEY_OPT="" + [ -n "${SSH_PROXY_KEY:-}" ] && PROXY_KEY_OPT="-i ${SSH_PROXY_KEY} " + SSH_EXTRA_ARGS="${SSH_EXTRA_ARGS} -o ProxyCommand=\"ssh ${PROXY_KEY_OPT}-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p ${SSH_PROXY_JUMP}\"" +fi + USE_DOCKER=true USE_LOG=true TARGET_HOST="" @@ -95,13 +106,14 @@ PSSH_COMMON=( -t 0 "${PSSH_TARGET_ARGS[@]}" "${PSSH_USER_ARGS[@]}" - -x "-i ${KeyFilePath}" + -x "${SSH_EXTRA_ARGS}" -O "$SSH_OPTION" ) if $USE_DOCKER; then INNER="set -euxo pipefail;" INNER+=" cd /root/mscclpp;" + INNER+=" [ -f /root/mscclpp/.ldpath ] && source /root/mscclpp/.ldpath;" INNER+=" export LD_LIBRARY_PATH=/root/mscclpp/build/lib:\\\$LD_LIBRARY_PATH;" INNER+=" CMD_B64='${CMD_B64}';" INNER+=" TMP=\\\$(mktemp);" diff --git a/test/deploy/setup.sh b/test/deploy/setup.sh index 0cb6f4b86..15e8e179a 100644 --- a/test/deploy/setup.sh +++ b/test/deploy/setup.sh @@ -31,6 +31,7 @@ fi make -C /root/mscclpp/tools/peer-access-test set +e /root/mscclpp/tools/peer-access-test/peer_access_test + PEER_ACCESS_EXIT_CODE=$? set -e if [ ${PEER_ACCESS_EXIT_CODE} -eq 2 ] && [ "${PLATFORM}" == "cuda" ]; then @@ -48,6 +49,7 @@ if [ ${PEER_ACCESS_EXIT_CODE} -eq 2 ] && [ "${PLATFORM}" == "cuda" ]; then elif [ ${PEER_ACCESS_EXIT_CODE} -ne 0 ]; then exit ${PEER_ACCESS_EXIT_CODE} fi + make -C /root/mscclpp/tools/peer-access-test clean if [ "${PLATFORM}" == "rocm" ]; then @@ -76,4 +78,4 @@ pip3 install setuptools_scm python3 -m setuptools_scm --force-write-version-files mkdir -p /var/run/sshd -/usr/sbin/sshd -p 22345 +/usr/sbin/sshd -p 22345 \ No newline at end of file