Stress | Move to ci/stress and build against sqlclient-ci-package output

.github/instructions/ado-pipelines.instructions.md

@@ -22,7 +22,7 @@ Top-level CI/PR pipeline files:
 - `dotnet-sqlclient-ci-project-reference-pipeline.yml` — CI with Project references (Release)
 - `sqlclient-pr-package-ref-pipeline.yml` — PR validation with Package references
 - `sqlclient-pr-project-ref-pipeline.yml` — PR validation with Project references
-- `stress/stress-tests-pipeline.yml` — Stress test pipeline and templates
+- `ci/stress/sqlclient-ci-stress-pipeline.yml` — Stress test pipeline and templates
 
 Reusable templates are organized under:
 - `common/templates/jobs/` — Job templates (`ci-build-nugets-job`, `ci-code-coverage-job`, `ci-run-tests-job`)
@@ -63,7 +63,7 @@ Stages execute in dependency order (Package reference mode requires artifacts fr
 7. `verify_nuget_packages_stage` — Verify NuGet package metadata
 8. `ci_run_tests_stage` — Run MDS and AKV test suites
 
-Stress testing is no longer a stage threaded through `dotnet-sqlclient-ci-core.yml`; it lives under `eng/pipelines/stress/` as a separate pipeline flow.
+Stress testing is no longer a stage threaded through `dotnet-sqlclient-ci-core.yml`; it lives under `eng/pipelines/ci/stress/` as a separate pipeline flow.
 
 When adding a new build stage, respect the dependency graph and pass artifact names/versions to downstream stages.
 

.github/prompts/update-build-pipelines.prompt.md

@@ -22,7 +22,7 @@ Follow this workflow step-by-step:
   - `sqlclient-pr-package-ref-pipeline.yml` — PR validation (package references)
   - `onebranch/sqlclient-official.yml` — official OneBranch build/release pipeline
   - `onebranch/sqlclient-non-official.yml` — non-official OneBranch build/release pipeline
-  - `stress/stress-tests-pipeline.yml` — stress test pipeline
+  - `ci/stress/sqlclient-ci-stress-pipeline.yml` — stress test pipeline
 - Shared templates live in `eng/pipelines/common/templates/` (jobs/, stages/, steps/).
 - CI variables are defined in `eng/pipelines/libraries/`; OneBranch variables are defined in `eng/pipelines/onebranch/variables/`.
 
@@ -57,7 +57,7 @@ Follow this workflow step-by-step:
 - Test filters by platform: `nonnetfxtests`, `nonnetcoreapptests`, `nonwindowstests`, `nonlinuxtests`.
 - SNI testing matrix: both Native (`useManagedSNI=false`) and Managed (`useManagedSNI=true`).
 - Always Encrypted tests controlled by `runAlwaysEncryptedTests` parameter.
-- Stress coverage is maintained under `eng/pipelines/stress/`, not as a stage inside `dotnet-sqlclient-ci-core.yml`.
+- Stress coverage is maintained under `eng/pipelines/ci/stress/`, not as a stage inside `dotnet-sqlclient-ci-core.yml`.
 
 ## 5. Validate
 - Verify YAML syntax is valid.

eng/pipelines/stress/stress-tests-job.yml → eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml

@@ -3,7 +3,10 @@
 # file to you under the MIT license.  See the LICENSE file in the project root for more information.
 ####################################################################################################
 
-# This job builds and runs stress tests by compiling the SqlClient projects transitively.
+# This job builds and runs stress tests against the SqlClient NuGet packages produced by the
+# sqlclient-ci-package pipeline.  The package artifacts are downloaded into the local NuGet feed
+# (packages/) and the stress test projects are built in "Package" mode (ReferenceType=Package)
+# against the exact package versions produced by that pipeline.
 #
 # The stress tests are located here:
 #
@@ -59,10 +62,6 @@ parameters:
   - name: netTestRuntimes
     type: object
 
-  # The name of the Azure Pipelines pool to use.
-  - name: poolName
-    type: string
-
   # The local SQL Server instance's 'sa' password, for use in the config file.
   - name: saPassword
     type: string
@@ -72,23 +71,26 @@ parameters:
   - name: sqlSetupStep
     type: step
 
-  # The pool VM image to use.
+  # The pool VM image to use.  Both the internal (ADO-1ES-Pool) and public (ADO-CI-1ES-Pool) pools
+  # must provide an image with this name.
+  #
+  # NOTE: This value is evaluated at template-expansion (compile) time to select the pool, so it
+  # must be a literal and must not contain any runtime expressions (e.g. $(...) macros or
+  # $[...] runtime expressions).
   - name: vmImage
     type: string
 
 jobs:
   - job: stress_tests_job_${{ parameters.jobNameSuffix }}
     displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests'
-    pool:
-      name: ${{ parameters.poolName }}
-      ${{ if eq(parameters.poolName, 'Azure Pipelines') }}:
-        vmImage: ${{ parameters.vmImage }}
-      ${{ else }}:
-        demands:
-          - imageOverride -equals ${{ parameters.vmImage }}
 
     variables:
 
+      # Whether this is an internal (ADO.Net project) or public (Public project) build.  Evaluated
+      # at compile time so it can drive template expressions below.
+      - name: isInternalBuild
+        value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }}
+
       # Import the variable group that provides SQL Server build properties used by the
       # shared configure-sql-server-*-step.yml templates (e.g. x64AliasRegistryPath,
       # x86AliasRegistryPath, SQLAliasName, SQLAliasPort).
@@ -97,6 +99,11 @@ jobs:
       # Import the variable group that provides SQL Server test configuration variables.
       - group: ADO Test Configuration Properties
 
+      # The directory where the "- download:" step places the SqlClient-Driver-Packages artifact
+      # published by the triggering sqlclient-ci-package pipeline.
+      - name: packageArtifactDir
+        value: $(Pipeline.Workspace)/sqlclient-ci-package/SqlClient-Driver-Packages
+
       # The top-level project file to build and run.
       - name: project
         value: $(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj
@@ -128,23 +135,50 @@ jobs:
       # bypassing the "--" separator.  This causes app arguments to contain SDK options and triggers
       # unintended behavior.
 
+      # Reference-mode arguments shared by build and run.  The stress tests are built in "Package"
+      # mode against the SqlClient packages downloaded from the sqlclient-ci-package pipeline.  The
+      # exact versions are resolved at runtime from the downloaded .nupkg filenames (see the
+      # "Stage Packages and Resolve Versions" step).
+      - name: referenceArgs
+        value: >-
+          -p:ReferenceType=Package
+          -p:SqlClientPackageVersion=$(mdsPackageVersion)
+          -p:AzurePackageVersion=$(azurePackageVersion)
+
       # dotnet CLI options for build.
       - name: dotnetBuildOpts
         value: >-
           --verbosity ${{ parameters.dotnetVerbosity }}
           -p:Configuration=${{ parameters.buildConfiguration }}
+          $(referenceArgs)
 
       # dotnet run options shared by all test steps (framework is appended per-step).
       - name: dotnetRunOpts
         value: >-
           --no-build
           --verbosity ${{ parameters.dotnetVerbosity }}
           --configuration ${{ parameters.buildConfiguration }}
+          $(referenceArgs)
 
       # Stress test options passed after the "--" separator.
       - name: stressTestOpts
         value: --assembly SqlClient.Stress.Tests --console
 
+    # Select the pool based on the requested image:
+    #   - macOS images run on the Microsoft-hosted 'Azure Pipelines' pool (no 1ES macOS images).
+    #   - All other images run on the 1ES pool for the current ADO project, matched via imageOverride.
+    pool:
+      ${{ if startsWith(parameters.vmImage, 'macos') }}:
+        name: 'Azure Pipelines'
+        vmImage: ${{ parameters.vmImage }}
+      ${{ else }}:
+        ${{ if eq(variables.isInternalBuild, true) }}:
+          name: ADO-1ES-Pool
+        ${{ else }}:
+          name: ADO-CI-1ES-Pool
+        demands:
+          - imageOverride -equals ${{ parameters.vmImage }}
+
     steps:
 
       # Install the .NET SDK and Runtimes.
@@ -170,9 +204,47 @@ jobs:
             # Write the JSON content to the config file.
             $content | Out-File -FilePath "config.jsonc"
 
+      # Download the SqlClient driver packages published by the triggering sqlclient-ci-package
+      # pipeline into the pipeline workspace.
+      - download: sqlclient-ci-package
+        artifact: SqlClient-Driver-Packages
+        displayName: Download SqlClient Driver Packages
+
+      # Copy the downloaded packages into the local NuGet feed (packages/) and resolve the exact
+      # Microsoft.Data.SqlClient and Microsoft.Data.SqlClient.Extensions.Azure versions from the
+      # .nupkg filenames.  These versions are passed to the build via the referenceArgs variable.
+      - task: PowerShell@2
+        displayName: Stage Packages and Resolve Versions
+        inputs:
+          pwsh: true
+          targetType: inline
+          script: |
+            $ErrorActionPreference = 'Stop'
+            $feed = "$(Build.SourcesDirectory)/packages"
+            New-Item -ItemType Directory -Force -Path $feed | Out-Null
+
+            Copy-Item "$(packageArtifactDir)/*.nupkg" $feed -Force
+            Copy-Item "$(packageArtifactDir)/*.snupkg" $feed -Force -ErrorAction SilentlyContinue
+
+            $mdsPattern = '^Microsoft\.Data\.SqlClient\.(\d[^\\/]*)\.nupkg$'
+            $azurePattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Azure\.(\d[^\\/]*)\.nupkg$'
+
+            $mds = Get-ChildItem "$feed/*.nupkg" | Where-Object { $_.Name -match $mdsPattern } | Select-Object -First 1
+            $azure = Get-ChildItem "$feed/*.nupkg" | Where-Object { $_.Name -match $azurePattern } | Select-Object -First 1
+
+            if (-not $mds) { throw "Microsoft.Data.SqlClient package not found in $feed" }
+            if (-not $azure) { throw "Microsoft.Data.SqlClient.Extensions.Azure package not found in $feed" }
+
+            $mdsVersion = [regex]::Match($mds.Name, $mdsPattern).Groups[1].Value
+            $azureVersion = [regex]::Match($azure.Name, $azurePattern).Groups[1].Value
+
+            Write-Host "Resolved Microsoft.Data.SqlClient version: $mdsVersion"
+            Write-Host "Resolved Microsoft.Data.SqlClient.Extensions.Azure version: $azureVersion"
+            Write-Host "##vso[task.setvariable variable=mdsPackageVersion]$mdsVersion"
+            Write-Host "##vso[task.setvariable variable=azurePackageVersion]$azureVersion"
+
       # Authenticate with NuGet feeds so that upstream packages (e.g. runtime host packs) can be
-      # fetched through the ADO Artifacts feed.  This is required on hosted pool agents (macOS)
-      # that do not have pre-configured feed credentials.
+      # fetched through the ADO Artifacts feed.
       - task: NuGetAuthenticate@1
         displayName: Authenticate NuGet feeds
 

eng/pipelines/stress/stress-tests-pipeline.yml → eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml

@@ -4,17 +4,15 @@
 ####################################################################################################
 
 # This pipeline runs the stress test suite against the SqlClient projects, building them
-# transitively as necessary, triggered by successful runs of the following pipelines:
+# transitively as necessary, triggered by successful runs of the sqlclient-ci-package pipeline:
 #
 # Public project:
-#   Triggering pipelines: PR-SqlClient-Project, CI-SqlClient (branch main only)
-#   Pipeline name:        sqlclient-stress
-#   Pipeline URL:         https://sqlclientdrivers.visualstudio.com/public/_build?definitionId=2250
+#   Triggering pipeline:  sqlclient-ci-package
+#   Pipeline name:        sqlclient-ci-stress
 #
 # ADO.Net project:
-#   Triggering pipeline:  MDS Main CI (branch internal/main only)
-#   Pipeline name:        sqlclient-stress
-#   Pipeline URL:         https://dev.azure.com/SqlClientDrivers/ADO.Net/_build?definitionId=2284
+#   Triggering pipeline:  sqlclient-ci-package
+#   Pipeline name:        sqlclient-ci-stress
 
 # Set the pipeline run name to the day-of-year and the daily run counter.
 name: $(DayOfYear)$(Rev:rr)
@@ -23,38 +21,31 @@ name: $(DayOfYear)$(Rev:rr)
 pr: none
 trigger: none
 
-# Trigger this pipeline after successful runs of the desired pipelines.
+# Trigger this pipeline after successful runs of the sqlclient-ci-package pipeline.
 #
 # The pipeline identifiers are displayed in the Azure DevOps UI, so it is helpful if they indicate
 # the project, folder, and pipeline name, hence the verbose values below.
 #
 resources:
   pipelines:
 
-    # The PR-SqlClient-Project pipeline in the Public project.
-    - pipeline: Public-ADO-PR-SqlClient-Project
-      project: Public
-      source: /ADO/PR-SqlClient-Project
+    # Trigger this pipeline when the sqlclient-ci-package pipeline completes successfully.
+    #
+    # 'project' is omitted so the resource resolves to the sqlclient-ci-package pipeline in the
+    # *current* ADO project.  This scopes the trigger per project automatically: the Public
+    # registration is triggered only by Public's sqlclient-ci-package, and the ADO.Net registration
+    # only by ADO.Net's.
+    #
+    # IMPORTANT: 'source' is the bare pipeline name with no folder path.  This REQUIRES the
+    # sqlclient-ci-package name to be UNIQUE within BOTH the Public and ADO.Net projects.  Azure
+    # DevOps only allows omitting the folder when the name is unambiguous; if a second pipeline with
+    # this name is ever added to either project, this resource will fail to resolve and the folder
+    # path must be added back (which, because the folders differ per project, would then require a
+    # per-project approach instead of this single shared definition).
+    - pipeline: sqlclient-ci-package
+      source: sqlclient-ci-package
       trigger: true
 
-    # The CI-SqlClient pipeline in the Public project.
-    - pipeline: Public-ADO-CI-SqlClient
-      project: Public
-      source: /ADO/CI-SqlClient
-      trigger:
-        branches:
-          include:
-            - main
-
-    # The MDS Main CI pipeline in the ADO.Net project.
-    - pipeline: ADO-Net-Internal-CI-MDS-Main-CI
-      project: ADO.Net
-      source: /CI/MDS Main CI
-      trigger:
-        branches:
-          include:
-            - internal/main
-
 # Pipeline parameters, visible in the Azure DevOps UI.
 parameters:
 
@@ -92,17 +83,6 @@ parameters:
       - detailed
       - diagnostic
 
-# Pipeline-level variables evaluated at compile time.
-variables:
-  # The 1ES pool name, determined automatically by ADO project:
-  #   - ADO.Net  -> ADO-1ES-Pool
-  #   - public   -> ADO-CI-1ES-Pool
-  - name: poolName
-    ${{ if eq(variables['System.TeamProject'], 'ADO.Net') }}:
-      value: ADO-1ES-Pool
-    ${{ else }}:
-      value: ADO-CI-1ES-Pool
-
 # The stages to run.
 stages:
 
@@ -112,10 +92,9 @@ stages:
       debug: ${{ parameters.debug }}
 
   # Run the stress tests.
-  - template: /eng/pipelines/stress/stress-tests-stage.yml@self
+  - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self
     parameters:
       buildConfiguration: ${{ parameters.buildConfiguration }}
       debug: ${{ parameters.debug }}
-      poolName: ${{ variables.poolName }}
       warnOnTestFailure: ${{ parameters.warnOnTestFailure }}
       dotnetVerbosity: ${{ parameters.dotnetVerbosity }}

eng/pipelines/stress/stress-tests-stage.yml → eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml

@@ -57,10 +57,6 @@ parameters:
     type: object
     default: [net8.0, net9.0, net10.0]
 
-  # The 1ES pool name for Linux and Windows jobs.  macOS always uses 'Azure Pipelines'.
-  - name: poolName
-    type: string
-
 stages:
   - stage: stress_tests_stage
     displayName: Run Stress Tests
@@ -77,7 +73,7 @@ stages:
     # ----------------------------------------------------------------------------------------------
     # Build and test on Linux.
 
-    - template: /eng/pipelines/stress/stress-tests-job.yml@self
+    - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self
       parameters:
         buildConfiguration: ${{ parameters.buildConfiguration }}
         debug: ${{ parameters.debug }}
@@ -88,7 +84,6 @@ stages:
         # No .NET Framework runtimes on Linux.
         netFrameworkTestRuntimes: []
         netTestRuntimes: ${{ parameters.netTestRuntimes }}
-        poolName: ${{ parameters.poolName }}
         saPassword: $(saPassword)
         sqlSetupStep:
           template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self
@@ -99,7 +94,7 @@ stages:
     # ----------------------------------------------------------------------------------------------
     # Build and test on Windows
 
-    - template: /eng/pipelines/stress/stress-tests-job.yml@self
+    - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self
       parameters:
         buildConfiguration: ${{ parameters.buildConfiguration }}
         debug: ${{ parameters.debug }}
@@ -110,7 +105,6 @@ stages:
         # Note that we include the .NET Framework runtimes for test runs on Windows.
         netFrameworkTestRuntimes: ${{ parameters.netFrameworkTestRuntimes }}
         netTestRuntimes: ${{ parameters.netTestRuntimes }}
-        poolName: ${{ parameters.poolName }}
         saPassword: $(saPassword)
         sqlSetupStep:
           template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self
@@ -123,7 +117,7 @@ stages:
     # ----------------------------------------------------------------------------------------------
     # Build and test on macOS.
 
-    - template: /eng/pipelines/stress/stress-tests-job.yml@self
+    - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self
       parameters:
         buildConfiguration: ${{ parameters.buildConfiguration }}
         debug: ${{ parameters.debug }}
@@ -134,12 +128,10 @@ stages:
         # No .NET Framework runtimes on macOS.
         netFrameworkTestRuntimes: []
         netTestRuntimes: ${{ parameters.netTestRuntimes }}
-        # We don't have any 1ES Hosted Pool images for macOS, so we use a generic one from Azure
-        # Pipelines.
-        poolName: Azure Pipelines
         saPassword: $(saPassword)
         sqlSetupStep:
           template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self
           parameters:
             saPassword: $(saPassword)
+        # A macOS vmImage name routes this job to the Microsoft-hosted 'Azure Pipelines' pool.
         vmImage: macos-latest

eng/pipelines/sqlclient-pr-package-ref-pipeline.yml

@@ -54,7 +54,6 @@ pr:
       - eng/pipelines/ci/*
       - eng/pipelines/kerberos/*
       - eng/pipelines/onebranch/*
-      - eng/pipelines/stress/*
 
 # Do not trigger commit or schedule runs for this pipeline.
 trigger: none

eng/pipelines/sqlclient-pr-project-ref-pipeline.yml

@@ -54,7 +54,6 @@ pr:
       - eng/pipelines/ci/*
       - eng/pipelines/kerberos/*
       - eng/pipelines/onebranch/*
-      - eng/pipelines/stress/*
 
 # Do not trigger commit or schedule runs for this pipeline.
 trigger: none