From 90c6d2855c9c2a52445868087e399b9455bf8f15 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 15:26:23 -0700 Subject: [PATCH 01/90] Beginning to consolidate the pipelines. Only trying it for the CI pipeline currently. Lots of commented bits. Only trying Windows builds. --- .vsts-ci.yml | 450 ++++++++---------- .../build-configuration.json | 16 +- eng/build.yml | 15 +- eng/pipelines/templates/jobs/sdk-build.yml | 131 +++++ .../templates/jobs/sdk-job-matrix.yml | 210 ++++++++ .../templates/variables/sdk-defaults.yml | 21 + 6 files changed, 576 insertions(+), 267 deletions(-) create mode 100644 eng/pipelines/templates/jobs/sdk-build.yml create mode 100644 eng/pipelines/templates/jobs/sdk-job-matrix.yml create mode 100644 eng/pipelines/templates/variables/sdk-defaults.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e187ad568963..a10ed5d2969b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -16,38 +16,32 @@ pr: - release/* - internal/release/* +parameters: +- name: runTests + displayName: Run tests + type: boolean + default: false + variables: - - name: _CIBuild - value: -restore -build -sign -pack -ci - # Disable post-build signing for internal release-branch builds or internal manual builds. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - - name: PostBuildSign - value: false - - ${{ else }}: - - name: PostBuildSign - value: true - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: _InternalRuntimeDownloadArgs - value: '' - - name: _OfficialBuildArgs - value: '' - - name: "skipComponentGovernanceDetection" - value: "true" - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - name: _OfficialBuildArgs - value: /p:OfficialBuilder=Microsoft - - name: Codeql.Enabled - value: true - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-CLI-SDLValidation-Params - - template: /eng/common/templates-official/variables/pool-providers.yml - # Set the MicroBuild plugin installation directory to the agent temp directory to avoid SDL tool scanning. - - name: MicroBuildOutputFolderOverride - value: $(Agent.TempDirectory) +- template: /eng/pipelines/templates/variables/sdk-defaults.yml +# Variables used: DncEngInternalBuildPool +- template: /eng/common/templates-official/variables/pool-providers.yml +# Variables used: dotnetbuilds-internal-container-read-token-base64 +- group: DotNetBuilds storage account read tokens +- name: runtimeSourceProperties + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) +# Helix testing requires a token when internally run. +# Variables used: HelixApiAccessToken +- group: DotNet-HelixApi-Access +# Runs a signed build by disabling post-build signing for release-branch builds or manual builds. +- ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: + - name: PostBuildSign + value: false +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - group: DotNet-CLI-SDLValidation-Params +# Set the MicroBuild plugin installation directory to the agent temp directory to avoid SDL tool scanning. +# - name: MicroBuildOutputFolderOverride +# value: $(Agent.TempDirectory) resources: repositories: @@ -57,255 +51,211 @@ resources: ref: refs/tags/release extends: - template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines parameters: sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows + # componentgovernance: + # # Setting to Critical as aspnet templates currently cause a High alert (default alert level is High). + # # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 + # # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ + # alertWarningLevel: Critical stages: - stage: build displayName: Build jobs: - - job: Publish_Build_Configuration - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # - job: Publish_Build_Configuration + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-windows-2022 + # os: windows + # steps: + # - task: 1ES.PublishPipelineArtifact@1 + # displayName: Publish Build Config + # inputs: + # targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration + # artifactName: buildConfiguration + + #################### SOURCE BUILD #################### + + # - template: /eng/common/templates-official/job/source-build.yml@self + # parameters: + # platform: + # name: Managed + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + + #################### BUILD #################### + - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + pool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - steps: - - task: 1ES.PublishPipelineArtifact@1 - displayName: Publish Build Config - inputs: - targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration - artifactName: buildConfiguration - - template: /eng/build.yml@self - parameters: - agentOs: Windows_NT - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - variables: - - name: _BuildConfig - value: Release - - name: _PublishArgs - value: '-publish /p:DotNetPublishUsingPipelines=true' - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - name: _SignType - value: test - - name: _Test - value: -test - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _SignType - value: real - - name: _Test - value: '' - - template: /eng/common/templates-official/job/source-build.yml@self - parameters: - platform: - name: 'Managed' - container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build.yml@self - parameters: - agentOs: Windows_NT_FullFramework - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - variables: - - name: _BuildConfig - value: Debug - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - name: _Test - value: -test + helixTargetQueue: Windows.Amd64.VS2022.Pre + runTests: ${{ parameters.runTests }} + 1espt: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + # containers: + # alpine319WithNode: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + # cblMariner20Fpm: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + # centosStream9: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + # debian11Amd64: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + # fedora39: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + # ubuntu2204: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + # mariner20CrossArm: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + # ubuntu2204DebPkg: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + strategy: + matrix: + ${{ if and(not(parameters.runTests), ne(variables['Build.Reason'], 'PullRequest')) }}: + Official: + publishArgument: -publish + officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=real /p:TeamName=DotNetCore + ${{ else }}: + Test: {} + FullFramework: + testFullMSBuild: true + TestAsTools: + testInBuildProperties: /p:RunTestsAsTool=true + TemplateEngine: + testName: TemplateEngineTests + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + AoT: + runAoTTests: true - - template: /eng/build.yml@self - parameters: - agentOs: Windows_NT_TestAsTools - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - variables: - - name: _BuildConfig - value: Debug - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - template: /eng/build.yml@self - parameters: - agentOs: Ubuntu_22_04 - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-ubuntu-2204-open - os: linux - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: ubuntu.2204.amd64.open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Ubuntu.2204.Amd64 - variables: - - name: _BuildConfig - value: Release - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - name: _Test - value: -test - - template: /eng/build.yml@self - parameters: - agentOs: Darwin - pool: - name: Azure Pipelines - image: macOS-latest - os: macOS - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.13.Amd64.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.13.Amd64 - variables: - - name: _BuildConfig - value: Release - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - name: _Test - value: -test + # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + # # dotnet-format builds + # - template: /eng/dotnet-format/dotnet-format-integration.yml@self + # parameters: + # PublishTaskName: 1ES.PublishBuildArtifacts@1 - # template-engine builds - - template: /eng/build.yml@self - parameters: - agentOs: Windows_NT_TemplateEngine - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - variables: - - name: _BuildConfig - value: Release - - name: PublishArgs - value: '-publish /p:DotNetPublishUsingPipelines=true' - - name: _SignType - value: test - - name: _Test - value: -test + # - template: /eng/build.yml@self + # parameters: + # agentOs: Ubuntu_22_04 + # pool: + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # name: $(DncEngPublicBuildPool) + # image: 1es-ubuntu-2204-open + # os: linux + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # name: $(DncEngInternalBuildPool) + # image: 1es-ubuntu-2204 + # os: linux + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: ubuntu.2204.amd64.open + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: Ubuntu.2204.Amd64 + # variables: + # - name: _BuildConfig + # value: Release + # - name: _PublishArgs + # value: '' + # - name: _SignType + # value: test + # - name: _Test + # value: -test - - template: /eng/build.yml@self - parameters: - agentOs: Ubuntu_22_04_TemplateEngine - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: 1es-ubuntu-2204-open - os: linux - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: 'ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64' - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Ubuntu.2204.Amd64 - variables: - - name: _BuildConfig - value: Release - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - name: _Test - value: -test + # - template: /eng/build.yml@self + # parameters: + # agentOs: Darwin + # pool: + # name: Azure Pipelines + # image: macOS-latest + # os: macOS + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: OSX.13.Amd64.Open + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: OSX.13.Amd64 + # variables: + # - name: _BuildConfig + # value: Release + # - name: _PublishArgs + # value: '' + # - name: _SignType + # value: test + # - name: _Test + # value: -test - - template: /eng/build.yml@self - parameters: - agentOs: Darwin_TemplateEngine - pool: - name: Azure Pipelines - image: macOS-latest - os: macOS - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.1100.Amd64.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.1100.Amd64 - variables: - - name: _BuildConfig - value: Release - - name: _PublishArgs - value: '' - - name: _SignType - value: test - - name: _Test - value: -test + # # template-engine builds + # - template: /eng/build.yml@self + # parameters: + # agentOs: Ubuntu_22_04_TemplateEngine + # pool: + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # name: $(DncEngPublicBuildPool) + # image: 1es-ubuntu-2204-open + # os: linux + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # name: $(DncEngInternalBuildPool) + # image: 1es-ubuntu-2204 + # os: linux + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: 'ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64' + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: Ubuntu.2204.Amd64 + # variables: + # - name: _BuildConfig + # value: Release + # - name: _PublishArgs + # value: '' + # - name: _SignType + # value: test + # - name: _Test + # value: -test - # dotnet-format builds - - template: /eng/dotnet-format/dotnet-format-integration.yml@self - parameters: - PublishTaskName: 1ES.PublishBuildArtifacts@1 + # - template: /eng/build.yml@self + # parameters: + # agentOs: Darwin_TemplateEngine + # pool: + # name: Azure Pipelines + # image: macOS-latest + # os: macOS + # ${{ if eq(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: OSX.1100.Amd64.Open + # ${{ if ne(variables['System.TeamProject'], 'public') }}: + # helixTargetQueue: OSX.1100.Amd64 + # variables: + # - name: _BuildConfig + # value: Release + # - name: _PublishArgs + # value: '' + # - name: _SignType + # value: test + # - name: _Test + # value: -test - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/templates-official/job/publish-build-assets.yml@self parameters: publishUsingPipelines: true publishAssetsImmediately: true dependsOn: - - Windows_NT - - Source_Build_Managed + - Windows_NT + - Source_Build_Managed pool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: publishingInfraVersion: 3 diff --git a/eng/BuildConfiguration/build-configuration.json b/eng/BuildConfiguration/build-configuration.json index b845ab40883a..5b698979faab 100644 --- a/eng/BuildConfiguration/build-configuration.json +++ b/eng/BuildConfiguration/build-configuration.json @@ -1,10 +1,10 @@ { - "RetryCountLimit":0, - "RetryByPipeline":{ - "RetryStages":[ - { - "StageName":"build" - } - ] - } + "RetryCountLimit": 0, + "RetryByPipeline": { + "RetryStages": [ + { + "StageName": "build" + } + ] + } } diff --git a/eng/build.yml b/eng/build.yml index fcb11ebcc940..d54effe04852 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -1,22 +1,19 @@ parameters: # Agent OS identifier and used as job name agentOs: '' - # Agent pool pool: {} - # Additional variables variables: {} - # Build strategy - matrix strategy: '' - # Job timeout timeoutInMinutes: 180 jobs: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0.3xx'), not(contains(parameters.agentOs, 'TemplateEngine'))) }}: - - template: /eng/common/templates-official/job/onelocbuild.yml@self +- ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0.3xx'), not(contains(parameters.agentOs, 'TemplateEngine'))) }}: + # Refdoc: https://github.com/dotnet/arcade/blob/main/Documentation/OneLocBuild.md#onelocbuild-template-parameters + - template: /eng/common/templates-official/job/onelocbuild.yml parameters: CreatePr: true LclSource: lclFilesfromPackage @@ -25,7 +22,7 @@ jobs: MirrorRepo: sdk - ${{ if not(contains(parameters.agentOs, 'TemplateEngine')) }}: - - template: /eng/common/templates-official/job/job.yml@self + - template: /eng/common/templates-official/job/job.yml parameters: name: ${{ parameters.agentOs }} enableMicrobuild: true @@ -212,7 +209,7 @@ jobs: # AoT Jobs - ${{ if and(in(parameters.agentOs, 'Windows_NT', 'Darwin'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}: - - template: /eng/common/templates-official/job/job.yml@self + - template: /eng/common/templates-official/job/job.yml parameters: name: ${{ parameters.agentOs }}_AoT_Tests enableMicrobuild: true @@ -359,7 +356,7 @@ jobs: # TemplateEngine Jobs - ${{ if contains(parameters.agentOs, 'TemplateEngine') }}: - - template: /eng/common/templates-official/job/job.yml@self + - template: /eng/common/templates-official/job/job.yml parameters: name: ${{ parameters.agentOs }} enableMicrobuild: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml new file mode 100644 index 000000000000..c701d689c284 --- /dev/null +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -0,0 +1,131 @@ +parameters: + runTests: false + 1espt: + jobTemplatePath: /eng/common/templates/job/job.yml + publishTaskName: PublishBuildArtifacts@1 + # buildCategory: Build + buildArchitecture: x64 + # # Agent OS identifier and used as job name + # agentOs: '' + # # Agent pool + # pool: {} + # # Additional variables + # variables: {} + # # Build strategy - matrix + # strategy: '' + +jobs: +- template: ${{ parameters.1espt.jobTemplatePath }} + parameters: + name: $(System.PhaseName)_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: '$(System.PhaseName): ${{ parameters.pool.os }} [${{ parameters.buildArchitecture }}]' + pool: ${{ parameters.pool }} + strategy: ${{ parameters.strategy }} + helixRepo: dotnet/sdk + timeoutInMinutes: 180 + enableMicrobuild: true + enablePublishBuildAssets: true + enableTelemetry: true + enablePublishUsingPipelines: true + # variables: + # - ${{ insert }}: ${{ parameters.variables }} + + steps: + ############### WINDOWS ############### + - ${{ if eq(parameters.pool.os, 'windows') }}: + ############### BUILDING ############### + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - powershell: eng/common/build.ps1 + -restore -build -sign -pack -ci -nativeToolsOnMachine + -configuration $(buildConfiguration) + $(runtimeSourceProperties) + $(testInBuildProperties) + $(officialBuildProperties) + displayName: Build + env: + # Required by Arcade + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + BuildConfig: $(buildConfiguration) + TestFullMSBuild: $(testFullMSBuild) + + ############### TESTING ############### + - ${{ if or(parameters.runTests, eq(variables['Build.Reason'], 'PullRequest')) }}: + - powershell: eng/common/build.ps1 + -restore -test -ci -nativeToolsOnMachine -prepareMachine + -configuration $(buildConfiguration) + -projects $(testProjects) + $(runtimeSourceProperties) + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + displayName: Run $(testName) + condition: succeeded() + env: + # Required by Arcade + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: $(runAoTTests) + + + + + + # - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: + # - script: eng/common/build.sh + # $(_CIBuild) + # --configuration $(_BuildConfig) + # $(_SignArgs) + # $(_OfficialBuildIdArgs) + # $(_InternalRuntimeDownloadArgs) + # -p:Test=false + # continueOnError: false + # displayName: Build + # env: + # BuildConfig: $(_BuildConfig) + # - powershell: eng/common/build.sh + # -configuration $(_BuildConfig) + # -prepareMachine + # -ci + # -restore + # -test + # -projects $(Build.SourcesDirectory)/test/UnitTests.proj + # /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog + # /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + # $(_InternalRuntimeDownloadArgs) + # displayName: Run Tests in Helix + # condition: succeededOrFailed() + # env: + # SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # HelixAccessToken: $(_HelixApiToken) + # RunAoTTests: 'false' + + + # TODO: Do we need to copy logs? Can't we just publish them directly? + - task: CopyFiles@2 + displayName: Copy Logs + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts + Contents: | + log/$(buildConfiguration)/**/* + TestResults/$(buildConfiguration)/**/* + SymStore/$(buildConfiguration)/**/* + tmp/$(buildConfiguration)/**/*.binlog + TargetFolder: $(Build.ArtifactStagingDirectory) + continueOnError: true + condition: always() + + - task: ${{ parameters.1espt.publishTaskName }} + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: $(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber) + publishLocation: Container + continueOnError: true + condition: always() diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml new file mode 100644 index 000000000000..e31e2f30ebb7 --- /dev/null +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -0,0 +1,210 @@ +parameters: + jobTemplatePath: /eng/common/templates/job/job.yml + publishTaskName: PublishBuildArtifacts@1 + buildCategory: Build + buildArchitecture: x64 + # Agent OS identifier and used as job name + agentOs: '' + # Agent pool + pool: {} + # Additional variables + variables: {} + # Build strategy - matrix + strategy: '' + + testFullMSBuild: false + publishArgs: '' + signType: test + buildConfiguration: Debug + testArgs: -test + additionalArgs: '' + +jobs: +- template: ${{ parameters.jobTemplatePath }} + parameters: + name: ${{ parameters.buildCategory }}_${{ parameters.agentOs }}_${{ parameters.buildArchitecture }} + displayName: '${{ parameters.buildCategory }}: ${{ parameters.agentOs }} [${{ parameters.buildArchitecture }}]' + enableMicrobuild: true + enablePublishBuildAssets: true + enableTelemetry: true + enablePublishUsingPipelines: true + helixRepo: dotnet/sdk + pool: ${{ parameters.pool }} + timeoutInMinutes: 180 + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + # workspace: + # clean: all + variables: + - ${{ insert }}: ${{ parameters.variables }} + # - _AgentOSName: ${{ parameters.agentOs }} + - _TeamName: DotNetCore + - _OfficialBuildIdArgs: '' + - _SignArgs: '' + - _HelixApiToken: '' + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + # Helix Testing requires a token when internally run + - group: DotNet-HelixApi-Access + - _HelixApiToken: $(HelixApiAccessToken) + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: dotnet-benchview + - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + - _PerfIterations: 25 + + steps: + ############## BUILDING ############## + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - powershell: eng\common\build.ps1 + $(_CIBuild) + -configuration $(_BuildConfig) + -nativeToolsOnMachine + $(_PublishArgs) + $(_SignArgs) + $(_OfficialBuildIdArgs) + /p:Test=false + $(_InternalRuntimeDownloadArgs) + $(_OfficialBuildArgs) + displayName: Build + env: + BuildConfig: $(_BuildConfig) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: ${{ parameters.HelixAccessToken }} + + ############## TESTING ############## + - powershell: eng\common\build.ps1 + -configuration $(_BuildConfig) + -nativeToolsOnMachine + -prepareMachine + -ci + -restore + $(_Test) + -projects $(Build.SourcesDirectory)\test\UnitTests.proj + /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + $(_InternalRuntimeDownloadArgs) + displayName: Run Tests in Helix + condition: succeeded() + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(_HelixApiToken) + RunAoTTests: 'false' + + + + + + + + + + + + + + + + + + - ${{ if eq(parameters.agentOs, 'Windows_NT_FullFramework') }}: + - powershell: eng\common\build.ps1 + $(_CIBuild) + -configuration $(_BuildConfig) + -nativeToolsOnMachine + $(_SignArgs) + $(_OfficialBuildIdArgs) + $(_InternalRuntimeDownloadArgs) + /p:Test=false + continueOnError: false + displayName: Build + env: + BuildConfig: $(_BuildConfig) + TestFullMSBuild: 'true' + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: ${{ parameters.HelixAccessToken }} + - powershell: eng\common\build.ps1 + -configuration $(_BuildConfig) + -nativeToolsOnMachine + -prepareMachine + -ci + -restore + -test + -projects $(Build.SourcesDirectory)\test\UnitTests.proj + /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + $(_InternalRuntimeDownloadArgs) + displayName: Run Tests in Helix + condition: succeededOrFailed() + env: + TestFullMSBuild: 'true' + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(_HelixApiToken) + + - ${{ if eq(parameters.agentOs, 'Windows_NT_TestAsTools') }}: + - powershell: eng\common\build.ps1 + $(_CIBuild) + -configuration $(_BuildConfig) + -nativeToolsOnMachine + $(_SignArgs) + $(_OfficialBuildIdArgs) + $(_InternalRuntimeDownloadArgs) + /p:RunTestsAsTool=true + displayName: Build + env: + BuildConfig: $(_BuildConfig) + + - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: + - script: eng/common/build.sh + $(_CIBuild) + --configuration $(_BuildConfig) + $(_SignArgs) + $(_OfficialBuildIdArgs) + $(_InternalRuntimeDownloadArgs) + -p:Test=false + continueOnError: false + displayName: Build + env: + BuildConfig: $(_BuildConfig) + - powershell: eng/common/build.sh + -configuration $(_BuildConfig) + -prepareMachine + -ci + -restore + -test + -projects $(Build.SourcesDirectory)/test/UnitTests.proj + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + $(_InternalRuntimeDownloadArgs) + displayName: Run Tests in Helix + condition: succeededOrFailed() + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(_HelixApiToken) + RunAoTTests: 'false' + + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + Contents: | + log/$(_BuildConfig)/**/* + TestResults/$(_BuildConfig)/**/* + SymStore/$(_BuildConfig)/**/* + tmp/$(_BuildConfig)/**/*.binlog + TargetFolder: '$(Build.ArtifactStagingDirectory)' + continueOnError: true + condition: always() + + - task: ${{ parameters.publishTaskName }} + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' + publishLocation: Container + continueOnError: true + condition: always() diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml new file mode 100644 index 000000000000..5869b875063d --- /dev/null +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -0,0 +1,21 @@ +variables: + buildConfiguration: Release + testFullMSBuild: false + runAoTTests: false + + testInBuildProperties: /p:Test=false + testName: HelixTests + testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj + + runtimeSourceProperties: '' + + Codeql.ExcludePathPatterns: '.packages, artifacts' + HelixApiAccessToken: '' + + # "Post-build signing" means that this build will not sign itself. + # This value is used by Arcade. + PostBuildSign: true + + + publishArgument: '' + officialBuildProperties: '' From 4314cbffd20b63f892f1a341f438117eebef49e5 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 15:29:48 -0700 Subject: [PATCH 02/90] Invalid syntax for parameter object. --- .vsts-ci.yml | 5 ++--- eng/pipelines/templates/jobs/sdk-build.yml | 9 ++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a10ed5d2969b..276b19b9f31b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -100,9 +100,8 @@ extends: os: windows helixTargetQueue: Windows.Amd64.VS2022.Pre runTests: ${{ parameters.runTests }} - 1espt: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 # containers: # alpine319WithNode: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index c701d689c284..d70d4e493bb8 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,8 +1,7 @@ parameters: runTests: false - 1espt: - jobTemplatePath: /eng/common/templates/job/job.yml - publishTaskName: PublishBuildArtifacts@1 + jobTemplatePath: /eng/common/templates/job/job.yml + publishTaskName: PublishBuildArtifacts@1 # buildCategory: Build buildArchitecture: x64 # # Agent OS identifier and used as job name @@ -15,7 +14,7 @@ parameters: # strategy: '' jobs: -- template: ${{ parameters.1espt.jobTemplatePath }} +- template: ${{ parameters.jobTemplatePath }} parameters: name: $(System.PhaseName)_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} displayName: '$(System.PhaseName): ${{ parameters.pool.os }} [${{ parameters.buildArchitecture }}]' @@ -121,7 +120,7 @@ jobs: continueOnError: true condition: always() - - task: ${{ parameters.1espt.publishTaskName }} + - task: ${{ parameters.publishTaskName }} displayName: Publish Logs to VSTS inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) From 8bb9f606d3dd0f7e4f8cb5d437b1c82fc4f35b3e Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 15:34:58 -0700 Subject: [PATCH 03/90] Trying to access a variable properly. --- eng/pipelines/templates/jobs/sdk-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index d70d4e493bb8..695c15f65426 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -16,8 +16,8 @@ parameters: jobs: - template: ${{ parameters.jobTemplatePath }} parameters: - name: $(System.PhaseName)_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: '$(System.PhaseName): ${{ parameters.pool.os }} [${{ parameters.buildArchitecture }}]' + name: ${{ variables['System.PhaseName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: '${{ variables[''System.PhaseName''] }}: ${{ parameters.pool.os }} [${{ parameters.buildArchitecture }}]' pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk From 13806cabf999b72664326865a2d9feee81942986 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 15:36:18 -0700 Subject: [PATCH 04/90] Commenting out extra stuff for now. --- .vsts-ci.yml | 73 ++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 276b19b9f31b..9a1299788bd6 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -241,38 +241,43 @@ extends: # - name: _Test # value: -test - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates-official/job/publish-build-assets.yml@self - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - dependsOn: - - Windows_NT - - Source_Build_Managed - pool: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates-official/post-build/post-build.yml@self - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "dotnet-sdk" - -TsaCodebaseName "dotnet-sdk" - -TsaPublish $True' + + + + + # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # - template: /eng/common/templates-official/job/publish-build-assets.yml@self + # parameters: + # publishUsingPipelines: true + # publishAssetsImmediately: true + # dependsOn: + # - Windows_NT + # - Source_Build_Managed + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-windows-2022 + # os: windows + + # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # - template: /eng/common/templates-official/post-build/post-build.yml@self + # parameters: + # publishingInfraVersion: 3 + # enableSymbolValidation: false + # enableSigningValidation: false + # enableNugetValidation: false + # enableSourceLinkValidation: false + # publishInstallersAndChecksums: true + # publishAssetsImmediately: true + # SDLValidationParameters: + # enable: false + # params: ' -SourceToolsList @("policheck","credscan") + # -TsaInstanceURL $(_TsaInstanceURL) + # -TsaProjectName $(_TsaProjectName) + # -TsaNotificationEmail $(_TsaNotificationEmail) + # -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + # -TsaBugAreaPath $(_TsaBugAreaPath) + # -TsaIterationPath $(_TsaIterationPath) + # -TsaRepositoryName "dotnet-sdk" + # -TsaCodebaseName "dotnet-sdk" + # -TsaPublish $True' From 578acea187a9e0c008253a24c267ce8ea6bd7f00 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 16:30:29 -0700 Subject: [PATCH 05/90] Changed runTests to be runtime variable and only modify the condition of the task. Forgot publishArgument for the build. Testing stuff with PhaseName. --- .vsts-ci.yml | 11 +++-- eng/pipelines/templates/jobs/sdk-build.yml | 46 +++++++++---------- .../templates/variables/sdk-defaults.yml | 1 + 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 9a1299788bd6..75eeb472b790 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -18,7 +18,7 @@ pr: parameters: - name: runTests - displayName: Run tests + displayName: Run Tests type: boolean default: false @@ -99,9 +99,10 @@ extends: image: 1es-windows-2022 os: windows helixTargetQueue: Windows.Amd64.VS2022.Pre - runTests: ${{ parameters.runTests }} jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 + publishTaskName: '1ES.PublishBuildArtifacts@1' + variables: + runTests: ${{ parameters.runTests }} # containers: # alpine319WithNode: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode @@ -121,7 +122,7 @@ extends: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg strategy: matrix: - ${{ if and(not(parameters.runTests), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: Official: publishArgument: -publish officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=real /p:TeamName=DotNetCore @@ -131,6 +132,8 @@ extends: testFullMSBuild: true TestAsTools: testInBuildProperties: /p:RunTestsAsTool=true + # This uses the build step for testing, so the explicit test step is not necessary. + runTests: false TemplateEngine: testName: TemplateEngineTests testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 695c15f65426..220eaf390a6e 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,7 +1,6 @@ parameters: - runTests: false jobTemplatePath: /eng/common/templates/job/job.yml - publishTaskName: PublishBuildArtifacts@1 + publishTaskName: 'PublishBuildArtifacts@1' # buildCategory: Build buildArchitecture: x64 # # Agent OS identifier and used as job name @@ -16,8 +15,8 @@ parameters: jobs: - template: ${{ parameters.jobTemplatePath }} parameters: - name: ${{ variables['System.PhaseName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: '${{ variables[''System.PhaseName''] }}: ${{ parameters.pool.os }} [${{ parameters.buildArchitecture }}]' + name: ${{ variables.System.PhaseName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: ${{ variables['System.PhaseName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk @@ -42,6 +41,7 @@ jobs: - powershell: eng/common/build.ps1 -restore -build -sign -pack -ci -nativeToolsOnMachine -configuration $(buildConfiguration) + $(publishArgument) $(runtimeSourceProperties) $(testInBuildProperties) $(officialBuildProperties) @@ -55,22 +55,21 @@ jobs: TestFullMSBuild: $(testFullMSBuild) ############### TESTING ############### - - ${{ if or(parameters.runTests, eq(variables['Build.Reason'], 'PullRequest')) }}: - - powershell: eng/common/build.ps1 - -restore -test -ci -nativeToolsOnMachine -prepareMachine - -configuration $(buildConfiguration) - -projects $(testProjects) - $(runtimeSourceProperties) - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - displayName: Run $(testName) - condition: succeeded() - env: - # Required by Arcade - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - RunAoTTests: $(runAoTTests) + - powershell: eng/common/build.ps1 + -restore -test -ci -nativeToolsOnMachine -prepareMachine + -configuration $(buildConfiguration) + -projects $(testProjects) + $(runtimeSourceProperties) + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + displayName: Run $(testName) + condition: and(succeeded(), eq(variables.runTests, true)) + env: + # Required by Arcade + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: $(runAoTTests) @@ -106,7 +105,6 @@ jobs: # RunAoTTests: 'false' - # TODO: Do we need to copy logs? Can't we just publish them directly? - task: CopyFiles@2 displayName: Copy Logs inputs: @@ -121,10 +119,10 @@ jobs: condition: always() - task: ${{ parameters.publishTaskName }} - displayName: Publish Logs to VSTS + displayName: Publish Logs inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory) - ArtifactName: $(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber) + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '${{ parameters.pool.os }}_$(Agent.JobName)_$(Build.BuildNumber)' publishLocation: Container continueOnError: true condition: always() diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 5869b875063d..e2d324c9867e 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,5 +1,6 @@ variables: buildConfiguration: Release + runTests: true testFullMSBuild: false runAoTTests: false From a3b37957e8e693c50aab5956ed172c50b2ea694a Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 16:38:32 -0700 Subject: [PATCH 06/90] Does this work for PhaseName? --- eng/pipelines/templates/jobs/sdk-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 220eaf390a6e..599496fc1480 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -15,8 +15,8 @@ parameters: jobs: - template: ${{ parameters.jobTemplatePath }} parameters: - name: ${{ variables.System.PhaseName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: ${{ variables['System.PhaseName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} + name: $(variables['System.PhaseName'])_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: $(variables['System.PhaseName']) ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk From 5287d25577cacfc1ac1d6413c553a5d2f9d03172 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 16:41:00 -0700 Subject: [PATCH 07/90] What does this do? --- .vsts-ci.yml | 1 + eng/pipelines/templates/jobs/sdk-build.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 75eeb472b790..8c2e7074b22b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -101,6 +101,7 @@ extends: helixTargetQueue: Windows.Amd64.VS2022.Pre jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: '1ES.PublishBuildArtifacts@1' + phaseName: $(System.PhaseName) variables: runTests: ${{ parameters.runTests }} # containers: diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 599496fc1480..d3795a7e5734 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -15,8 +15,8 @@ parameters: jobs: - template: ${{ parameters.jobTemplatePath }} parameters: - name: $(variables['System.PhaseName'])_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: $(variables['System.PhaseName']) ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} + name: ${{ parameters.phaseName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: ${{ parameters.phaseName }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk From 0cc1f46282d1cd9369c804ef97069f121695b532 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 16:58:35 -0700 Subject: [PATCH 08/90] Trying oneEsPt as object name for template parameter object. Trying System.JobName. --- .vsts-ci.yml | 8 ++++---- eng/pipelines/templates/jobs/sdk-build.yml | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 8c2e7074b22b..4421f76890fa 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -99,9 +99,9 @@ extends: image: 1es-windows-2022 os: windows helixTargetQueue: Windows.Amd64.VS2022.Pre - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: '1ES.PublishBuildArtifacts@1' - phaseName: $(System.PhaseName) + oneEsPt: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 variables: runTests: ${{ parameters.runTests }} # containers: @@ -133,7 +133,7 @@ extends: testFullMSBuild: true TestAsTools: testInBuildProperties: /p:RunTestsAsTool=true - # This uses the build step for testing, so the explicit test step is not necessary. + # This uses the build step for testing, so the extra test step is not necessary. runTests: false TemplateEngine: testName: TemplateEngineTests diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index d3795a7e5734..2cbc5139b8f0 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,6 +1,7 @@ parameters: - jobTemplatePath: /eng/common/templates/job/job.yml - publishTaskName: 'PublishBuildArtifacts@1' + oneEsPt: + jobTemplatePath: /eng/common/templates/job/job.yml + publishTaskName: PublishBuildArtifacts@1 # buildCategory: Build buildArchitecture: x64 # # Agent OS identifier and used as job name @@ -13,10 +14,10 @@ parameters: # strategy: '' jobs: -- template: ${{ parameters.jobTemplatePath }} +- template: ${{ parameters.oneEsPt.jobTemplatePath }} parameters: - name: ${{ parameters.phaseName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: ${{ parameters.phaseName }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} + name: ${{ variables['System.JobName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: ${{ variables['System.JobName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk @@ -118,11 +119,11 @@ jobs: continueOnError: true condition: always() - - task: ${{ parameters.publishTaskName }} + - task: ${{ parameters.oneEsPt.publishTaskName }} displayName: Publish Logs inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '${{ parameters.pool.os }}_$(Agent.JobName)_$(Build.BuildNumber)' + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: ${{ parameters.pool.os }}_$(Agent.JobName)_$(Build.BuildNumber) publishLocation: Container continueOnError: true condition: always() From 90bb9242464eb7fcfdbaa0bf9fdcd2971b05d326 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 17:08:26 -0700 Subject: [PATCH 09/90] Let's see if Agent.JobName does anything for fun. --- eng/pipelines/templates/jobs/sdk-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 2cbc5139b8f0..90fc421ad441 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -16,8 +16,8 @@ parameters: jobs: - template: ${{ parameters.oneEsPt.jobTemplatePath }} parameters: - name: ${{ variables['System.JobName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: ${{ variables['System.JobName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} + name: ${{ variables['Agent.JobName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: ${{ variables['Agent.JobName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk From 8bf4cb458799681eb9214f8653c92d885ac06159 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 17:53:02 -0700 Subject: [PATCH 10/90] Added ability to set buildArchitecture. Fixing signing issue with missing variable. --- .vsts-ci.yml | 19 +++++++++++++++-- eng/pipelines/templates/jobs/sdk-build.yml | 21 +++++++++++-------- .../templates/variables/sdk-defaults.yml | 12 ++++++----- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 4421f76890fa..7f4369514d7a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -34,9 +34,11 @@ variables: # Variables used: HelixApiAccessToken - group: DotNet-HelixApi-Access # Runs a signed build by disabling post-build signing for release-branch builds or manual builds. +# This is used by Arcade. - ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: - name: PostBuildSign value: false +# TODO: What is this for? - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - group: DotNet-CLI-SDLValidation-Params # Set the MicroBuild plugin installation directory to the agent temp directory to avoid SDL tool scanning. @@ -99,11 +101,13 @@ extends: image: 1es-windows-2022 os: windows helixTargetQueue: Windows.Amd64.VS2022.Pre - oneEsPt: + oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 variables: runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=$(_SignType) /p:TeamName=DotNetCore # containers: # alpine319WithNode: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode @@ -126,7 +130,18 @@ extends: ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: Official: publishArgument: -publish - officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=real /p:TeamName=DotNetCore + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_x86: + buildArchitecture: x86 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) ${{ else }}: Test: {} FullFramework: diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 90fc421ad441..7c94c5341e22 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,9 +1,9 @@ parameters: - oneEsPt: + oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 # buildCategory: Build - buildArchitecture: x64 + # buildArchitecture: x64 # # Agent OS identifier and used as job name # agentOs: '' # # Agent pool @@ -14,10 +14,10 @@ parameters: # strategy: '' jobs: -- template: ${{ parameters.oneEsPt.jobTemplatePath }} +- template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} parameters: - name: ${{ variables['Agent.JobName'] }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: ${{ variables['Agent.JobName'] }} ${{ parameters.pool.os }} ${{ parameters.buildArchitecture }} + name: ${{ parameters.pool.os }} + # displayName: ${{ parameters.pool.os }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk @@ -43,12 +43,13 @@ jobs: -restore -build -sign -pack -ci -nativeToolsOnMachine -configuration $(buildConfiguration) $(publishArgument) + /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) $(testInBuildProperties) $(officialBuildProperties) displayName: Build env: - # Required by Arcade + # Required by Arcade for passing to Helix SYSTEM_ACCESSTOKEN: $(System.AccessToken) # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) @@ -56,17 +57,19 @@ jobs: TestFullMSBuild: $(testFullMSBuild) ############### TESTING ############### + # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 -restore -test -ci -nativeToolsOnMachine -prepareMachine -configuration $(buildConfiguration) -projects $(testProjects) + /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog displayName: Run $(testName) condition: and(succeeded(), eq(variables.runTests, true)) env: - # Required by Arcade + # Required by Arcade for passing to Helix SYSTEM_ACCESSTOKEN: $(System.AccessToken) # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) @@ -119,11 +122,11 @@ jobs: continueOnError: true condition: always() - - task: ${{ parameters.oneEsPt.publishTaskName }} + - task: ${{ parameters.oneEsPtCompat.publishTaskName }} displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) - ArtifactName: ${{ parameters.pool.os }}_$(Agent.JobName)_$(Build.BuildNumber) + ArtifactName: ${{ parameters.pool.os }}_$(System.PhaseName) publishLocation: Container continueOnError: true condition: always() diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index e2d324c9867e..4b4e34dfebac 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,5 +1,6 @@ variables: buildConfiguration: Release + buildArchitecture: x64 runTests: true testFullMSBuild: false runAoTTests: false @@ -13,10 +14,11 @@ variables: Codeql.ExcludePathPatterns: '.packages, artifacts' HelixApiAccessToken: '' - # "Post-build signing" means that this build will not sign itself. - # This value is used by Arcade. - PostBuildSign: true - - publishArgument: '' officialBuildProperties: '' + + ############### ARCADE ############### + # "Post-build signing" means that this build will not sign itself. + PostBuildSign: true + # Both this (used in Arcade for the MicroBuildSigningPlugin) and DotNetSignType (used in Arcade in Sign.proj) are necessary to set the sign type. + _SignType: test \ No newline at end of file From 7fafb9395f8c8146356341a59d3c5ede041f3eaa Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 17:56:37 -0700 Subject: [PATCH 11/90] Passing the variables down. --- eng/pipelines/templates/jobs/sdk-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 7c94c5341e22..9ed81654fb67 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -26,8 +26,8 @@ jobs: enablePublishBuildAssets: true enableTelemetry: true enablePublishUsingPipelines: true - # variables: - # - ${{ insert }}: ${{ parameters.variables }} + variables: + - ${{ insert }}: ${{ parameters.variables }} steps: ############### WINDOWS ############### From 6fc8aee35d01758d5f6be785526eb807238c1ab2 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 18:31:45 -0700 Subject: [PATCH 12/90] In-progress for non-windows builds. Fixing signing using _TeamName. --- .vsts-ci.yml | 85 +++++++-- eng/pipelines/templates/jobs/sdk-build.yml | 170 ++++++++++-------- .../templates/variables/sdk-defaults.yml | 8 +- 3 files changed, 162 insertions(+), 101 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7f4369514d7a..831776037d72 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -100,31 +100,14 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - helixTargetQueue: Windows.Amd64.VS2022.Pre + helixTargetQueue: windows.amd64.vs2022.pre oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 variables: runTests: ${{ parameters.runTests }} # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=$(_SignType) /p:TeamName=DotNetCore - # containers: - # alpine319WithNode: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - # cblMariner20Fpm: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - # centosStream9: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - # debian11Amd64: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - # fedora39: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - # ubuntu2204: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - # mariner20CrossArm: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - # ubuntu2204DebPkg: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) strategy: matrix: ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -261,6 +244,70 @@ extends: # value: -test + # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + # parameters: + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-ubuntu-2204 + # os: linux + # helixTargetQueue: ubuntu.2204.amd64 + # oneEsPtCompat: + # jobTemplatePath: /eng/common/templates-official/job/job.yml + # publishTaskName: 1ES.PublishBuildArtifacts@1 + # variables: + # runTests: ${{ parameters.runTests }} + # # Setting this here once so it can be reused in the matrix only. + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=$(_SignType) /p:TeamName=DotNetCore + # containers: + # alpine319WithNode: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + # cblMariner20Fpm: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + # centosStream9: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + # debian11Amd64: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + # fedora39: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + # ubuntu2204: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + # mariner20CrossArm: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + # ubuntu2204DebPkg: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + # strategy: + # matrix: + # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Official: + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_x86: + # buildArchitecture: x86 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_arm64: + # buildArchitecture: arm64 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # ${{ else }}: + # Test: {} + # FullFramework: + # testFullMSBuild: true + # TestAsTools: + # testInBuildProperties: /p:RunTestsAsTool=true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # TemplateEngine: + # testName: TemplateEngineTests + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # AoT: + # runAoTTests: true + + diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 9ed81654fb67..87af4e443bd5 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -2,22 +2,13 @@ parameters: oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 - # buildCategory: Build - # buildArchitecture: x64 - # # Agent OS identifier and used as job name - # agentOs: '' - # # Agent pool - # pool: {} - # # Additional variables - # variables: {} - # # Build strategy - matrix - # strategy: '' + buildScriptPath: eng/common/build.ps1 jobs: - template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} parameters: name: ${{ parameters.pool.os }} - # displayName: ${{ parameters.pool.os }} + displayName: ${{ parameters.pool.os }} pool: ${{ parameters.pool }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk @@ -31,88 +22,109 @@ jobs: steps: ############### WINDOWS ############### - - ${{ if eq(parameters.pool.os, 'windows') }}: - ############### BUILDING ############### - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - displayName: Setup Private Feeds Credentials - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - powershell: eng/common/build.ps1 - -restore -build -sign -pack -ci -nativeToolsOnMachine - -configuration $(buildConfiguration) - $(publishArgument) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(testInBuildProperties) - $(officialBuildProperties) - displayName: Build + # - ${{ if eq(parameters.pool.os, 'windows') }}: + ############### BUILDING ############### + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + displayName: Setup Private Feeds Credentials env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - BuildConfig: $(buildConfiguration) - TestFullMSBuild: $(testFullMSBuild) - - ############### TESTING ############### - # TODO: Change _CustomHelixTargetQueue name - - powershell: eng/common/build.ps1 - -restore -test -ci -nativeToolsOnMachine -prepareMachine - -configuration $(buildConfiguration) - -projects $(testProjects) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - displayName: Run $(testName) - condition: and(succeeded(), eq(variables.runTests, true)) - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - RunAoTTests: $(runAoTTests) - + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - powershell: ${{ parameters.buildScriptPath }} + -restore -build -sign -pack -ci -nativeToolsOnMachine + -configuration $(buildConfiguration) + $(publishArgument) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + $(testInBuildProperties) + $(officialBuildProperties) + /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + displayName: Build + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + BuildConfig: $(buildConfiguration) + TestFullMSBuild: $(testFullMSBuild) + ############### TESTING ############### + # TODO: Change _CustomHelixTargetQueue name + - powershell: ${{ parameters.buildScriptPath }} + -restore -test -ci -nativeToolsOnMachine -prepareMachine + -configuration $(buildConfiguration) + -projects $(testProjects) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + displayName: Run $(testName) + condition: and(succeeded(), eq(variables.runTests, true)) + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: $(runAoTTests) + # ############### NON-WINDOWS ############### + # - ${{ else }}: + # ############### BUILDING ############### + # - ${{ if ne(variables['System.TeamProject'], 'public') }}: + # - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + # displayName: Setup Private Feeds Credentials + # env: + # Token: $(dn-bot-dnceng-artifact-feeds-rw) - # - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: - # - script: eng/common/build.sh - # $(_CIBuild) - # --configuration $(_BuildConfig) - # $(_SignArgs) - # $(_OfficialBuildIdArgs) - # $(_InternalRuntimeDownloadArgs) - # -p:Test=false - # continueOnError: false + # - powershell: eng/common/build.ps1 + # -restore -build -sign -pack -ci -nativeToolsOnMachine + # -configuration $(buildConfiguration) + # $(publishArgument) + # /p:Architecture=$(buildArchitecture) + # $(runtimeSourceProperties) + # $(testInBuildProperties) + # $(officialBuildProperties) # displayName: Build # env: - # BuildConfig: $(_BuildConfig) - # - powershell: eng/common/build.sh - # -configuration $(_BuildConfig) - # -prepareMachine - # -ci - # -restore - # -test - # -projects $(Build.SourcesDirectory)/test/UnitTests.proj - # /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - # /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - # $(_InternalRuntimeDownloadArgs) - # displayName: Run Tests in Helix - # condition: succeededOrFailed() + # # Required by Arcade for passing to Helix + # SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # # Only set when ran internally + # HelixAccessToken: $(HelixApiAccessToken) + # BuildConfig: $(buildConfiguration) + # TestFullMSBuild: $(testFullMSBuild) + + # ############### TESTING ############### + # # TODO: Change _CustomHelixTargetQueue name + # - powershell: eng/common/build.ps1 + # -restore -test -ci -nativeToolsOnMachine -prepareMachine + # -configuration $(buildConfiguration) + # -projects $(testProjects) + # /p:Architecture=$(buildArchitecture) + # $(runtimeSourceProperties) + # /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + # /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + # displayName: Run $(testName) + # condition: and(succeeded(), eq(variables.runTests, true)) # env: - # SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # HelixAccessToken: $(_HelixApiToken) - # RunAoTTests: 'false' + # # Required by Arcade for passing to Helix + # SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # # Only set when ran internally + # HelixAccessToken: $(HelixApiAccessToken) + # RunAoTTests: $(runAoTTests) + + + + + + + - task: CopyFiles@2 displayName: Copy Logs inputs: SourceFolder: $(Build.SourcesDirectory)/artifacts + # TODO: SymStore should not be copied in PR builds Contents: | log/$(buildConfiguration)/**/* TestResults/$(buildConfiguration)/**/* diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 4b4e34dfebac..ed7342778ea9 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -5,20 +5,22 @@ variables: testFullMSBuild: false runAoTTests: false + # TODO: Instead, just have this always on the build and set the value, same with RunTestsAsTool. testInBuildProperties: /p:Test=false testName: HelixTests testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj runtimeSourceProperties: '' - Codeql.ExcludePathPatterns: '.packages, artifacts' + Codeql.ExcludePathPatterns: artifacts, .packages HelixApiAccessToken: '' publishArgument: '' officialBuildProperties: '' ############### ARCADE ############### - # "Post-build signing" means that this build will not sign itself. + # "Post-build signing" means that this build will NOT sign itself. PostBuildSign: true # Both this (used in Arcade for the MicroBuildSigningPlugin) and DotNetSignType (used in Arcade in Sign.proj) are necessary to set the sign type. - _SignType: test \ No newline at end of file + _SignType: test + _TeamName: DotNetCore \ No newline at end of file From be2f78005cc859b9c00d5361f9a1f8cdc81a022d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 18:43:30 -0700 Subject: [PATCH 13/90] Trying to add an ubuntu run. --- .vsts-ci.yml | 129 +++++++++++---------- eng/pipelines/templates/jobs/sdk-build.yml | 3 + 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 831776037d72..f429d74bcf5d 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -58,6 +58,23 @@ extends: ${{ else }}: template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines parameters: + containers: + alpine319WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + cblMariner20Fpm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + centosStream9: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + debian11Amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + fedora39: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + ubuntu2204: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + mariner20CrossArm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + ubuntu2204DebPkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) @@ -93,7 +110,7 @@ extends: # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - #################### BUILD #################### + ############### WINDOWS ############### - template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: pool: @@ -243,69 +260,53 @@ extends: # - name: _Test # value: -test - - # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - # parameters: - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-ubuntu-2204 - # os: linux - # helixTargetQueue: ubuntu.2204.amd64 - # oneEsPtCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml - # publishTaskName: 1ES.PublishBuildArtifacts@1 - # variables: - # runTests: ${{ parameters.runTests }} - # # Setting this here once so it can be reused in the matrix only. - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetSignType=$(_SignType) /p:TeamName=DotNetCore - # containers: - # alpine319WithNode: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - # cblMariner20Fpm: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - # centosStream9: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - # debian11Amd64: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - # fedora39: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - # ubuntu2204: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - # mariner20CrossArm: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - # ubuntu2204DebPkg: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - # strategy: - # matrix: - # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Official: - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_x86: - # buildArchitecture: x86 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_arm64: - # buildArchitecture: arm64 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # ${{ else }}: - # Test: {} - # FullFramework: - # testFullMSBuild: true - # TestAsTools: - # testInBuildProperties: /p:RunTestsAsTool=true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # TemplateEngine: - # testName: TemplateEngineTests - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # AoT: - # runAoTTests: true + ############### NON-WINDOWS ############### + - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + variables: + runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + strategy: + matrix: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + Official: + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_x86: + buildArchitecture: x86 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + # ${{ else }}: + # Test: {} + # FullFramework: + # testFullMSBuild: true + # TestAsTools: + # testInBuildProperties: /p:RunTestsAsTool=true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # TemplateEngine: + # testName: TemplateEngineTests + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # AoT: + # runAoTTests: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 87af4e443bd5..494f2a2dd5d3 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -3,6 +3,7 @@ parameters: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 buildScriptPath: eng/common/build.ps1 + container: '' jobs: - template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} @@ -10,6 +11,8 @@ jobs: name: ${{ parameters.pool.os }} displayName: ${{ parameters.pool.os }} pool: ${{ parameters.pool }} + # ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk timeoutInMinutes: 180 From a3bf9389e79760f8cfba072ef9b3c32600d2538f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 19:03:20 -0700 Subject: [PATCH 14/90] The linux containers don't have powershell on them. Lovely. --- .vsts-ci.yml | 1 + eng/pipelines/templates/jobs/sdk-build.yml | 121 ++++++++++++++------- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index f429d74bcf5d..e99404557afd 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -272,6 +272,7 @@ extends: oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 + # buildScriptPath: eng/common/build.sh variables: runTests: ${{ parameters.runTests }} # Setting this here once so it can be reused in the matrix only. diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 494f2a2dd5d3..9f845774a6e3 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -2,7 +2,7 @@ parameters: oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 - buildScriptPath: eng/common/build.ps1 + # buildScriptPath: eng/common/build.ps1 container: '' jobs: @@ -28,47 +28,90 @@ jobs: # - ${{ if eq(parameters.pool.os, 'windows') }}: ############### BUILDING ############### - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - displayName: Setup Private Feeds Credentials + - ${{ if eq(parameters.pool.os, 'windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ else }}: + - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - ${{ if eq(parameters.pool.os, 'windows') }}: + - powershell: eng/common/build.ps1 + -restore -build -sign -pack -ci -nativeToolsOnMachine + -configuration $(buildConfiguration) + $(publishArgument) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + $(testInBuildProperties) + $(officialBuildProperties) + /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + displayName: Build env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - powershell: ${{ parameters.buildScriptPath }} - -restore -build -sign -pack -ci -nativeToolsOnMachine - -configuration $(buildConfiguration) - $(publishArgument) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(testInBuildProperties) - $(officialBuildProperties) - /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - displayName: Build - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - BuildConfig: $(buildConfiguration) - TestFullMSBuild: $(testFullMSBuild) + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + BuildConfig: $(buildConfiguration) + TestFullMSBuild: $(testFullMSBuild) + - ${{ else }}: + - script: eng/common/build.sh + -restore -build -sign -pack -ci -nativeToolsOnMachine + -configuration $(buildConfiguration) + $(publishArgument) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + $(testInBuildProperties) + $(officialBuildProperties) + /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + displayName: Build + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + BuildConfig: $(buildConfiguration) + TestFullMSBuild: $(testFullMSBuild) ############### TESTING ############### - # TODO: Change _CustomHelixTargetQueue name - - powershell: ${{ parameters.buildScriptPath }} - -restore -test -ci -nativeToolsOnMachine -prepareMachine - -configuration $(buildConfiguration) - -projects $(testProjects) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - displayName: Run $(testName) - condition: and(succeeded(), eq(variables.runTests, true)) - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - RunAoTTests: $(runAoTTests) + - ${{ if eq(parameters.pool.os, 'windows') }}: + # TODO: Change _CustomHelixTargetQueue name + - powershell: eng/common/build.ps1 + -restore -test -ci -nativeToolsOnMachine -prepareMachine + -configuration $(buildConfiguration) + -projects $(testProjects) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + displayName: Run $(testName) + condition: and(succeeded(), eq(variables.runTests, true)) + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: $(runAoTTests) + - ${{ else }}: + - script: eng/common/build.sh + -restore -test -ci -nativeToolsOnMachine -prepareMachine + -configuration $(buildConfiguration) + -projects $(testProjects) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog + displayName: Run $(testName) + condition: and(succeeded(), eq(variables.runTests, true)) + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: $(runAoTTests) # ############### NON-WINDOWS ############### # - ${{ else }}: From 442a51cf2e696eaa1e9241a4c069f2f1afa2ffa6 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 19:26:13 -0700 Subject: [PATCH 15/90] Try using bash instead of script task. --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 9f845774a6e3..eb0ff96bf228 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -34,7 +34,7 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ else }}: - - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + - bash: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token displayName: Setup Private Feeds Credentials env: Token: $(dn-bot-dnceng-artifact-feeds-rw) From 96724b27ea16a64805e0e8845c81a19f002e8894 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 May 2024 19:45:48 -0700 Subject: [PATCH 16/90] Do not do signing when running tests. --- .vsts-ci.yml | 4 ++-- eng/pipelines/templates/jobs/sdk-build.yml | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e99404557afd..b3e92a583e53 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -33,9 +33,9 @@ variables: # Helix testing requires a token when internally run. # Variables used: HelixApiAccessToken - group: DotNet-HelixApi-Access -# Runs a signed build by disabling post-build signing for release-branch builds or manual builds. +# Runs a signed build by disabling post-build signing for release-branch builds or manual builds that are not running tests. # This is used by Arcade. -- ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: +- ${{ if and(eq(parameters.runTests, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - name: PostBuildSign value: false # TODO: What is this for? diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index eb0ff96bf228..e68effb96416 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -26,7 +26,7 @@ jobs: steps: ############### WINDOWS ############### # - ${{ if eq(parameters.pool.os, 'windows') }}: - ############### BUILDING ############### + ############### PREP ############### - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token @@ -34,11 +34,12 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ else }}: - - bash: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token displayName: Setup Private Feeds Credentials env: Token: $(dn-bot-dnceng-artifact-feeds-rw) + ############### BUILDING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 -restore -build -sign -pack -ci -nativeToolsOnMachine @@ -165,7 +166,7 @@ jobs: - + ############### POST ############### - task: CopyFiles@2 displayName: Copy Logs inputs: From 17a702025dffebfbdf4a1422cb8ee2e92292efda Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 7 May 2024 15:01:54 -0700 Subject: [PATCH 17/90] Added RuntimeIdentifier variable. Added MacOS section. Commenting out SetupNugetSources for now. --- .vsts-ci.yml | 51 ++++++++++++++++++- eng/pipelines/templates/jobs/sdk-build.yml | 27 +++++----- .../templates/variables/sdk-defaults.yml | 11 ++-- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b3e92a583e53..298cecd7d6b1 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -260,7 +260,8 @@ extends: # - name: _Test # value: -test - ############### NON-WINDOWS ############### + ############### LINUX ############### + # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. - template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: pool: @@ -272,7 +273,6 @@ extends: oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 - # buildScriptPath: eng/common/build.sh variables: runTests: ${{ parameters.runTests }} # Setting this here once so it can be reused in the matrix only. @@ -309,6 +309,53 @@ extends: # AoT: # runAoTTests: true + ############### MACOS ############### + - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + pool: + name: Azure Pipelines + image: macOS-latest + os: macOS + helixTargetQueue: OSX.13.Amd64 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + variables: + runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + strategy: + matrix: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + Official: + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + runtimeIdentifierProperties: /p:Rid=osx-arm64 + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + + + + + + + + + + + + + + + + + + + diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index e68effb96416..cacf860b9a9e 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -2,7 +2,6 @@ parameters: oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 - # buildScriptPath: eng/common/build.ps1 container: '' jobs: @@ -27,17 +26,17 @@ jobs: ############### WINDOWS ############### # - ${{ if eq(parameters.pool.os, 'windows') }}: ############### PREP ############### - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if eq(parameters.pool.os, 'windows') }}: - - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - displayName: Setup Private Feeds Credentials - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ else }}: - - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token - displayName: Setup Private Feeds Credentials - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) + # - ${{ if ne(variables['System.TeamProject'], 'public') }}: + # - ${{ if eq(parameters.pool.os, 'windows') }}: + # - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + # displayName: Setup Private Feeds Credentials + # env: + # Token: $(dn-bot-dnceng-artifact-feeds-rw) + # - ${{ else }}: + # - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + # displayName: Setup Private Feeds Credentials + # env: + # Token: $(dn-bot-dnceng-artifact-feeds-rw) ############### BUILDING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: @@ -47,6 +46,7 @@ jobs: $(publishArgument) /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) + $(runtimeIdentifierProperties) $(testInBuildProperties) $(officialBuildProperties) /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) @@ -65,6 +65,7 @@ jobs: $(publishArgument) /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) + $(runtimeIdentifierProperties) $(testInBuildProperties) $(officialBuildProperties) /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) @@ -86,6 +87,7 @@ jobs: -projects $(testProjects) /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) + $(runtimeIdentifierProperties) /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog displayName: Run $(testName) @@ -103,6 +105,7 @@ jobs: -projects $(testProjects) /p:Architecture=$(buildArchitecture) $(runtimeSourceProperties) + $(runtimeIdentifierProperties) /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog displayName: Run $(testName) diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index ed7342778ea9..3243221d2acf 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,4 +1,5 @@ variables: + ############### BUILD ############### buildConfiguration: Release buildArchitecture: x64 runTests: true @@ -11,9 +12,7 @@ variables: testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj runtimeSourceProperties: '' - - Codeql.ExcludePathPatterns: artifacts, .packages - HelixApiAccessToken: '' + runtimeIdentifierProperties: '' publishArgument: '' officialBuildProperties: '' @@ -23,4 +22,8 @@ variables: PostBuildSign: true # Both this (used in Arcade for the MicroBuildSigningPlugin) and DotNetSignType (used in Arcade in Sign.proj) are necessary to set the sign type. _SignType: test - _TeamName: DotNetCore \ No newline at end of file + _TeamName: DotNetCore + HelixApiAccessToken: '' + + ############### OTHER ############### + Codeql.ExcludePathPatterns: artifacts, .packages \ No newline at end of file From 2294de35c189af2e692b49f1d5900b728e39f597 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 7 May 2024 15:04:17 -0700 Subject: [PATCH 18/90] Messed up build section alignment. --- eng/pipelines/templates/jobs/sdk-build.yml | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index cacf860b9a9e..f4dc5463200a 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -58,25 +58,25 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) TestFullMSBuild: $(testFullMSBuild) - - ${{ else }}: - - script: eng/common/build.sh - -restore -build -sign -pack -ci -nativeToolsOnMachine - -configuration $(buildConfiguration) - $(publishArgument) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(runtimeIdentifierProperties) - $(testInBuildProperties) - $(officialBuildProperties) - /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - displayName: Build - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - BuildConfig: $(buildConfiguration) - TestFullMSBuild: $(testFullMSBuild) + - ${{ else }}: + - script: eng/common/build.sh + -restore -build -sign -pack -ci -nativeToolsOnMachine + -configuration $(buildConfiguration) + $(publishArgument) + /p:Architecture=$(buildArchitecture) + $(runtimeSourceProperties) + $(runtimeIdentifierProperties) + $(testInBuildProperties) + $(officialBuildProperties) + /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + displayName: Build + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + BuildConfig: $(buildConfiguration) + TestFullMSBuild: $(testFullMSBuild) ############### TESTING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: From 7d2e73a6abc45af82d79835f37d0f8d00774aeac Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 7 May 2024 15:18:53 -0700 Subject: [PATCH 19/90] Remove nativeToolsOnMachine for non-windows. Set job name based on if container is set or not. --- eng/pipelines/templates/jobs/sdk-build.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index f4dc5463200a..1d35695b8284 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -7,10 +7,13 @@ parameters: jobs: - template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} parameters: - name: ${{ parameters.pool.os }} - displayName: ${{ parameters.pool.os }} + ${{ if eq(parameters.container, '') }}: + name: ${{ parameters.pool.os }} + displayName: ${{ parameters.pool.os }} + ${{ else }}: + name: ${{ parameters.pool.os }}_${{ parameters.container }} + displayName: ${{ parameters.pool.os }} [${{ parameters.container }}] pool: ${{ parameters.pool }} - # ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} strategy: ${{ parameters.strategy }} helixRepo: dotnet/sdk @@ -60,7 +63,7 @@ jobs: TestFullMSBuild: $(testFullMSBuild) - ${{ else }}: - script: eng/common/build.sh - -restore -build -sign -pack -ci -nativeToolsOnMachine + -restore -build -sign -pack -ci -configuration $(buildConfiguration) $(publishArgument) /p:Architecture=$(buildArchitecture) @@ -82,7 +85,7 @@ jobs: - ${{ if eq(parameters.pool.os, 'windows') }}: # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 - -restore -test -ci -nativeToolsOnMachine -prepareMachine + -restore -test -ci -prepareMachine -configuration $(buildConfiguration) -projects $(testProjects) /p:Architecture=$(buildArchitecture) From f7830978d548b1750db597372869d07358f3222f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:39:17 -0700 Subject: [PATCH 20/90] Converted everything to use compile-time parameters instead of matrixes with variables. Only trying Windows builds for the moment. Many smaller fixes, including names of things. --- .vsts-ci.yml | 451 +++++++++--------- eng/pipelines/templates/jobs/sdk-build.yml | 170 ++++--- .../templates/jobs/sdk-job-matrix.yml | 340 ++++++------- .../templates/variables/sdk-defaults.yml | 22 +- 4 files changed, 457 insertions(+), 526 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 298cecd7d6b1..1444f5a85654 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -20,7 +20,10 @@ parameters: - name: runTests displayName: Run Tests type: boolean - default: false + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + default: true + ${{ else }}: + default: false variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml @@ -28,22 +31,16 @@ variables: - template: /eng/common/templates-official/variables/pool-providers.yml # Variables used: dotnetbuilds-internal-container-read-token-base64 - group: DotNetBuilds storage account read tokens -- name: runtimeSourceProperties - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) # Helix testing requires a token when internally run. # Variables used: HelixApiAccessToken - group: DotNet-HelixApi-Access -# Runs a signed build by disabling post-build signing for release-branch builds or manual builds that are not running tests. -# This is used by Arcade. +# Allows Arcade to run a signed build by disabling post-build signing for release-branch builds or manual builds that are not running tests. - ${{ if and(eq(parameters.runTests, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - name: PostBuildSign value: false # TODO: What is this for? - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - group: DotNet-CLI-SDLValidation-Params -# Set the MicroBuild plugin installation directory to the agent temp directory to avoid SDL tool scanning. -# - name: MicroBuildOutputFolderOverride -# value: $(Agent.TempDirectory) resources: repositories: @@ -85,6 +82,47 @@ extends: # # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ # alertWarningLevel: Critical + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + ### LOCAL ONLY ### + _publishArgument: -publish + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + _windowsJobParameters: + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - ${{ else }}: + - categoryName: TestBuild + - categoryName: FullFramework + testFullMSBuild: true + - categoryName: TestAsTools + runTestsAsTool: true + # This uses the build step for testing, so the extra test step is not necessary. + runTests: false + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true + + stages: - stage: build displayName: Build @@ -101,242 +139,174 @@ extends: # targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration # artifactName: buildConfiguration - #################### SOURCE BUILD #################### - # - template: /eng/common/templates-official/job/source-build.yml@self + # - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self # parameters: - # platform: - # name: Managed - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-windows-2022 + # os: windows + # helixTargetQueue: windows.amd64.vs2022.pre + # ${{ if eq(parameters.runTests, false) }}: + # Official: + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_x86: + # buildArchitecture: x86 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_arm64: + # buildArchitecture: arm64 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # ${{ else }}: + # Test: {} + # FullFramework: + # testFullMSBuild: true + # TestAsTools: + # testInBuildProperties: /p:RunTestsAsTool=true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # TemplateEngine: + # testName: TemplateEngineTests + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # AoT: + # runAoTTests: true + ############### WINDOWS ############### - - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - helixTargetQueue: windows.amd64.vs2022.pre - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: + - ${{ each jobParameters in parameters._windowsJobParameters }}: + - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + ${{ insert }}: ${{ jobParameters }} + oneEsPtCompat: ${{ parameters.oneEsPtCompat }} runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_x86: - buildArchitecture: x86 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - ${{ else }}: - Test: {} - FullFramework: - testFullMSBuild: true - TestAsTools: - testInBuildProperties: /p:RunTestsAsTool=true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - TemplateEngine: - testName: TemplateEngineTests - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - AoT: - runAoTTests: true + # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + # parameters: + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-windows-2022 + # os: windows + # helixTargetQueue: windows.amd64.vs2022.pre + # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} + # # variables: + # # runTests: ${{ parameters.runTests }} + # # # Setting this here once so it can be reused in the matrix only. + # # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + # strategy: + # matrix: + # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Official: + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_x86: + # buildArchitecture: x86 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_arm64: + # buildArchitecture: arm64 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # ${{ else }}: + # Test: {} + # FullFramework: + # testFullMSBuild: true + # TestAsTools: + # testInBuildProperties: /p:RunTestsAsTool=true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # TemplateEngine: + # testName: TemplateEngineTests + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # AoT: + # runAoTTests: true + + + + + + + # ############### LINUX ############### + # # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. + # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + # parameters: + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-ubuntu-2204 + # os: linux + # helixTargetQueue: ubuntu.2204.amd64 + # container: ubuntu2204 + # oneEsPtCompat: + # jobTemplatePath: /eng/common/templates-official/job/job.yml + # publishTaskName: 1ES.PublishBuildArtifacts@1 + # variables: + # runTests: ${{ parameters.runTests }} + # # Setting this here once so it can be reused in the matrix only. + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + # strategy: + # matrix: + # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Official: + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_x86: + # buildArchitecture: x86 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_arm64: + # buildArchitecture: arm64 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # ${{ else }}: + # Test: {} + # Test_arm64: + # testFullMSBuild: true + + # ############### MACOS ############### + # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + # parameters: + # pool: + # name: Azure Pipelines + # image: macOS-latest + # os: macOS + # helixTargetQueue: OSX.13.Amd64 + # oneEsPtCompat: + # jobTemplatePath: /eng/common/templates-official/job/job.yml + # publishTaskName: 1ES.PublishBuildArtifacts@1 + # variables: + # runTests: ${{ parameters.runTests }} + # # Setting this here once so it can be reused in the matrix only. + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + # strategy: + # matrix: + # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Official: + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) + # Official_arm64: + # runtimeIdentifierProperties: /p:Rid=osx-arm64 + # buildArchitecture: arm64 + # publishArgument: -publish + # _SignType: real + # officialBuildProperties: $(_officialBuildProperties) - # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # # dotnet-format builds - # - template: /eng/dotnet-format/dotnet-format-integration.yml@self - # parameters: - # PublishTaskName: 1ES.PublishBuildArtifacts@1 - # - template: /eng/build.yml@self - # parameters: - # agentOs: Ubuntu_22_04 - # pool: - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # name: $(DncEngPublicBuildPool) - # image: 1es-ubuntu-2204-open - # os: linux - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # name: $(DncEngInternalBuildPool) - # image: 1es-ubuntu-2204 - # os: linux - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: ubuntu.2204.amd64.open - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: Ubuntu.2204.Amd64 - # variables: - # - name: _BuildConfig - # value: Release - # - name: _PublishArgs - # value: '' - # - name: _SignType - # value: test - # - name: _Test - # value: -test - - # - template: /eng/build.yml@self - # parameters: - # agentOs: Darwin - # pool: - # name: Azure Pipelines - # image: macOS-latest - # os: macOS - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: OSX.13.Amd64.Open - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: OSX.13.Amd64 - # variables: - # - name: _BuildConfig - # value: Release - # - name: _PublishArgs - # value: '' - # - name: _SignType - # value: test - # - name: _Test - # value: -test - - # # template-engine builds - # - template: /eng/build.yml@self - # parameters: - # agentOs: Ubuntu_22_04_TemplateEngine - # pool: - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # name: $(DncEngPublicBuildPool) - # image: 1es-ubuntu-2204-open - # os: linux - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # name: $(DncEngInternalBuildPool) - # image: 1es-ubuntu-2204 - # os: linux - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: 'ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64' - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: Ubuntu.2204.Amd64 - # variables: - # - name: _BuildConfig - # value: Release - # - name: _PublishArgs - # value: '' - # - name: _SignType - # value: test - # - name: _Test - # value: -test - - # - template: /eng/build.yml@self - # parameters: - # agentOs: Darwin_TemplateEngine - # pool: - # name: Azure Pipelines - # image: macOS-latest - # os: macOS - # ${{ if eq(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: OSX.1100.Amd64.Open - # ${{ if ne(variables['System.TeamProject'], 'public') }}: - # helixTargetQueue: OSX.1100.Amd64 - # variables: - # - name: _BuildConfig - # value: Release - # - name: _PublishArgs - # value: '' - # - name: _SignType - # value: test - # - name: _Test - # value: -test - - ############### LINUX ############### - # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. - - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: - runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_x86: - buildArchitecture: x86 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - # ${{ else }}: - # Test: {} - # FullFramework: - # testFullMSBuild: true - # TestAsTools: - # testInBuildProperties: /p:RunTestsAsTool=true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # TemplateEngine: - # testName: TemplateEngineTests - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # AoT: - # runAoTTests: true - - ############### MACOS ############### - - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: Azure Pipelines - image: macOS-latest - os: macOS - helixTargetQueue: OSX.13.Amd64 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: - runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - runtimeIdentifierProperties: /p:Rid=osx-arm64 - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) @@ -354,6 +324,19 @@ extends: + #################### SOURCE BUILD #################### + + # - template: /eng/common/templates-official/job/source-build.yml@self + # parameters: + # platform: + # name: Managed + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + # # dotnet-format builds + # - template: /eng/dotnet-format/dotnet-format-integration.yml@self + # parameters: + # PublishTaskName: 1ES.PublishBuildArtifacts@1 diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 1d35695b8284..b38bc148a7fb 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -3,16 +3,29 @@ parameters: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 container: '' + categoryName: Build + buildArchitecture: x64 + runTests: true + testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj + ### ENV VARS ### + testFullMSBuild: false + runAoTTests: false + ### MSBUILD ### + publishArgument: '' + runTestsAsTool: false + runtimeSourceProperties: '' + runtimeIdentifierProperties: '' + officialBuildProperties: '' jobs: - template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} parameters: ${{ if eq(parameters.container, '') }}: - name: ${{ parameters.pool.os }} - displayName: ${{ parameters.pool.os }} + name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + displayName: "${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }})" ${{ else }}: - name: ${{ parameters.pool.os }}_${{ parameters.container }} - displayName: ${{ parameters.pool.os }} [${{ parameters.container }}] + name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} + displayName: "${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }}) [${{ parameters.container }}]" pool: ${{ parameters.pool }} container: ${{ parameters.container }} strategy: ${{ parameters.strategy }} @@ -26,32 +39,30 @@ jobs: - ${{ insert }}: ${{ parameters.variables }} steps: - ############### WINDOWS ############### - # - ${{ if eq(parameters.pool.os, 'windows') }}: - ############### PREP ############### - # - ${{ if ne(variables['System.TeamProject'], 'public') }}: - # - ${{ if eq(parameters.pool.os, 'windows') }}: - # - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - # displayName: Setup Private Feeds Credentials - # env: - # Token: $(dn-bot-dnceng-artifact-feeds-rw) - # - ${{ else }}: - # - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token - # displayName: Setup Private Feeds Credentials - # env: - # Token: $(dn-bot-dnceng-artifact-feeds-rw) + ############## PREP ############### + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if eq(parameters.pool.os, 'windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ else }}: + - script: ./$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + displayName: Setup Private Feeds Credentials + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) ############### BUILDING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 -restore -build -sign -pack -ci -nativeToolsOnMachine -configuration $(buildConfiguration) - $(publishArgument) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(runtimeIdentifierProperties) - $(testInBuildProperties) - $(officialBuildProperties) + ${{ parameters.publishArgument }} + /p:Architecture=${{ parameters.buildArchitecture }} + /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} + ${{ parameters.runtimeSourceProperties }} + ${{ parameters.runtimeIdentifierProperties }} + ${{ parameters.officialBuildProperties }} /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: @@ -60,17 +71,17 @@ jobs: # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) - TestFullMSBuild: $(testFullMSBuild) + TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: - script: eng/common/build.sh -restore -build -sign -pack -ci -configuration $(buildConfiguration) - $(publishArgument) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(runtimeIdentifierProperties) - $(testInBuildProperties) - $(officialBuildProperties) + ${{ parameters.publishArgument }} + /p:Architecture=${{ parameters.buildArchitecture }} + /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} + ${{ parameters.runtimeSourceProperties }} + ${{ parameters.runtimeIdentifierProperties }} + ${{ parameters.officialBuildProperties }} /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: @@ -79,46 +90,47 @@ jobs: # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) - TestFullMSBuild: $(testFullMSBuild) + TestFullMSBuild: ${{ parameters.testFullMSBuild }} ############### TESTING ############### - - ${{ if eq(parameters.pool.os, 'windows') }}: - # TODO: Change _CustomHelixTargetQueue name - - powershell: eng/common/build.ps1 - -restore -test -ci -prepareMachine - -configuration $(buildConfiguration) - -projects $(testProjects) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(runtimeIdentifierProperties) - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - displayName: Run $(testName) - condition: and(succeeded(), eq(variables.runTests, true)) - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - RunAoTTests: $(runAoTTests) - - ${{ else }}: - - script: eng/common/build.sh - -restore -test -ci -nativeToolsOnMachine -prepareMachine - -configuration $(buildConfiguration) - -projects $(testProjects) - /p:Architecture=$(buildArchitecture) - $(runtimeSourceProperties) - $(runtimeIdentifierProperties) - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - displayName: Run $(testName) - condition: and(succeeded(), eq(variables.runTests, true)) - env: - # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) - RunAoTTests: $(runAoTTests) + - ${{ if eq(parameters.runTests, true) }}: + - ${{ if eq(parameters.pool.os, 'windows') }}: + # TODO: Change _CustomHelixTargetQueue name + - powershell: eng/common/build.ps1 + -restore -test -ci -prepareMachine -nativeToolsOnMachine + -configuration $(buildConfiguration) + -projects ${{ parameters.testProjects }} + /p:Architecture=${{ parameters.buildArchitecture }} + ${{ parameters.runtimeSourceProperties }} + ${{ parameters.runtimeIdentifierProperties }} + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog + displayName: Run ${{ parameters.categoryName }} Tests + condition: succeeded() + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: ${{ parameters.runAoTTests }} + - ${{ else }}: + - script: eng/common/build.sh + -restore -test -ci -prepareMachine + -configuration $(buildConfiguration) + -projects ${{ parameters.testProjects }} + /p:Architecture=${{ parameters.buildArchitecture }} + ${{ parameters.runtimeSourceProperties }} + ${{ parameters.runtimeIdentifierProperties }} + /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog + displayName: Run ${{ parameters.categoryName }} Tests + condition: succeeded() + env: + # Required by Arcade for passing to Helix + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # Only set when ran internally + HelixAccessToken: $(HelixApiAccessToken) + RunAoTTests: ${{ parameters.runAoTTests }} # ############### NON-WINDOWS ############### # - ${{ else }}: @@ -177,12 +189,17 @@ jobs: displayName: Copy Logs inputs: SourceFolder: $(Build.SourcesDirectory)/artifacts - # TODO: SymStore should not be copied in PR builds - Contents: | - log/$(buildConfiguration)/**/* - TestResults/$(buildConfiguration)/**/* - SymStore/$(buildConfiguration)/**/* - tmp/$(buildConfiguration)/**/*.binlog + ${{ if eq(parameters.runTests, false) }}: + Contents: | + log/$(buildConfiguration)/**/* + TestResults/$(buildConfiguration)/**/* + SymStore/$(buildConfiguration)/**/* + tmp/$(buildConfiguration)/**/*.binlog + ${{ else }}: + Contents: | + log/$(buildConfiguration)/**/* + TestResults/$(buildConfiguration)/**/* + tmp/$(buildConfiguration)/**/*.binlog TargetFolder: $(Build.ArtifactStagingDirectory) continueOnError: true condition: always() @@ -191,7 +208,8 @@ jobs: displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) - ArtifactName: ${{ parameters.pool.os }}_$(System.PhaseName) + # TODO: Figure out appropriate artifact name + ArtifactName: ${{ parameters.name }} publishLocation: Container continueOnError: true condition: always() diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index e31e2f30ebb7..4cacc119e088 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -1,210 +1,140 @@ parameters: - jobTemplatePath: /eng/common/templates/job/job.yml - publishTaskName: PublishBuildArtifacts@1 - buildCategory: Build - buildArchitecture: x64 - # Agent OS identifier and used as job name - agentOs: '' - # Agent pool - pool: {} - # Additional variables - variables: {} - # Build strategy - matrix - strategy: '' - - testFullMSBuild: false - publishArgs: '' - signType: test - buildConfiguration: Debug - testArgs: -test - additionalArgs: '' + # jobTemplatePath: /eng/common/templates/job/job.yml + # publishTaskName: PublishBuildArtifacts@1 + # buildCategory: Build + # buildArchitecture: x64 + # # Agent OS identifier and used as job name + # agentOs: '' + # # Agent pool + # pool: {} + # # Additional variables + # variables: {} + # # Build strategy - matrix + # strategy: '' + + # testFullMSBuild: false + # publishArgs: '' + # signType: test + # buildConfiguration: Debug + # testArgs: -test + # additionalArgs: '' + + + # linuxContainerInfo: + # ${{ if eq(parameters.runTests, false) }}: jobs: -- template: ${{ parameters.jobTemplatePath }} +############### WINDOWS ############### +- template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: - name: ${{ parameters.buildCategory }}_${{ parameters.agentOs }}_${{ parameters.buildArchitecture }} - displayName: '${{ parameters.buildCategory }}: ${{ parameters.agentOs }} [${{ parameters.buildArchitecture }}]' - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: 180 - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - # workspace: - # clean: all + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + helixTargetQueue: windows.amd64.vs2022.pre + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 variables: - - ${{ insert }}: ${{ parameters.variables }} - # - _AgentOSName: ${{ parameters.agentOs }} - - _TeamName: DotNetCore - - _OfficialBuildIdArgs: '' - - _SignArgs: '' - - _HelixApiToken: '' - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - # Helix Testing requires a token when internally run - - group: DotNet-HelixApi-Access - - _HelixApiToken: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - _PerfIterations: 25 - - steps: - ############## BUILDING ############## - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - displayName: Setup Private Feeds Credentials - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_PublishArgs) - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - $(_OfficialBuildArgs) - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - ############## TESTING ############## - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeeded() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - - - - - - - - - - - - - - - - - ${{ if eq(parameters.agentOs, 'Windows_NT_FullFramework') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - - - ${{ if eq(parameters.agentOs, 'Windows_NT_TestAsTools') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:RunTestsAsTool=true - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: - - script: eng/common/build.sh - $(_CIBuild) - --configuration $(_BuildConfig) - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/UnitTests.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - SymStore/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: ${{ parameters.publishTaskName }} - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() + runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + strategy: + matrix: + ${{ if eq(parameters.runTests, false) }}: + Official: + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_x86: + buildArchitecture: x86 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + ${{ else }}: + Test: {} + FullFramework: + testFullMSBuild: true + TestAsTools: + testInBuildProperties: /p:RunTestsAsTool=true + # This uses the build step for testing, so the extra test step is not necessary. + runTests: false + TemplateEngine: + testName: TemplateEngineTests + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + AoT: + runAoTTests: true + +############### LINUX ############### +# TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. +- template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + variables: + runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + strategy: + matrix: + ${{ if eq(parameters.runTests, false) }}: + Official: + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_x86: + buildArchitecture: x86 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + ${{ else }}: + Test: {} + Test_arm64: + testFullMSBuild: true + +############### MACOS ############### +- template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + pool: + name: Azure Pipelines + image: macOS-latest + os: macOS + helixTargetQueue: OSX.13.Amd64 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + variables: + runTests: ${{ parameters.runTests }} + # Setting this here once so it can be reused in the matrix only. + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + strategy: + matrix: + ${{ if eq(parameters.runTests, false) }}: + Official: + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) + Official_arm64: + runtimeIdentifierProperties: /p:Rid=osx-arm64 + buildArchitecture: arm64 + publishArgument: -publish + _SignType: real + officialBuildProperties: $(_officialBuildProperties) \ No newline at end of file diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 3243221d2acf..708b91503e02 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,21 +1,21 @@ variables: ############### BUILD ############### buildConfiguration: Release - buildArchitecture: x64 - runTests: true - testFullMSBuild: false - runAoTTests: false + # buildArchitecture: x64 + # runTests: true + # testFullMSBuild: false + # runAoTTests: false # TODO: Instead, just have this always on the build and set the value, same with RunTestsAsTool. - testInBuildProperties: /p:Test=false - testName: HelixTests - testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj + # testInBuildProperties: /p:Test=false + # testName: HelixTests + # testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj - runtimeSourceProperties: '' - runtimeIdentifierProperties: '' + # runtimeSourceProperties: '' + # runtimeIdentifierProperties: '' - publishArgument: '' - officialBuildProperties: '' + # publishArgument: '' + # officialBuildProperties: '' ############### ARCADE ############### # "Post-build signing" means that this build will NOT sign itself. From be09faa0b139c33436c16cbfba20d6a87c23f070 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:42:58 -0700 Subject: [PATCH 21/90] =?UTF-8?q?My=20test=20default=20doesn't=20work.=20?= =?UTF-8?q?=F0=9F=92=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vsts-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1444f5a85654..0052b5cfe756 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -20,10 +20,12 @@ parameters: - name: runTests displayName: Run Tests type: boolean - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - default: true - ${{ else }}: - default: false + default: false + # 💢 "A template expression is not allowed in this context" + # ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # default: true + # ${{ else }}: + # default: false variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml From 4f82464fd4e30ca84e608301ee082370bb24fd6f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:46:34 -0700 Subject: [PATCH 22/90] I think it doesn't like parameters that start with an underscore. --- .vsts-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 0052b5cfe756..a3fdf6e8cb38 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -89,25 +89,25 @@ extends: publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ### LOCAL ONLY ### - _publishArgument: -publish - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - _windowsJobParameters: + local_publishArgument: -publish + local_officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + local_windowsJobParameters: - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} + publishArgument: ${{ parameters.local_publishArgument }} + officialBuildProperties: ${{ parameters.local_officialBuildProperties }} variables: _SignType: real - categoryName: Official buildArchitecture: x86 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} + publishArgument: ${{ parameters.local_publishArgument }} + officialBuildProperties: ${{ parameters.local_officialBuildProperties }} variables: _SignType: real - categoryName: Official buildArchitecture: arm64 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} + publishArgument: ${{ parameters.local_publishArgument }} + officialBuildProperties: ${{ parameters.local_officialBuildProperties }} variables: _SignType: real - ${{ else }}: @@ -183,7 +183,7 @@ extends: ############### WINDOWS ############### - - ${{ each jobParameters in parameters._windowsJobParameters }}: + - ${{ each jobParameters in parameters.local_windowsJobParameters }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: ${{ insert }}: ${{ jobParameters }} From 3ee96ef75d1adbe4869a93cec4b49d3ae794aa11 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:49:35 -0700 Subject: [PATCH 23/90] Just testing this. --- .vsts-ci.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a3fdf6e8cb38..2f6da79d8013 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,6 +26,9 @@ parameters: # default: true # ${{ else }}: # default: false +- name: _publishArgument + type: string + default: -publish variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml @@ -89,25 +92,25 @@ extends: publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ### LOCAL ONLY ### - local_publishArgument: -publish - local_officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - local_windowsJobParameters: + # _publishArgument: -publish + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + _windowsJobParameters: - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official - publishArgument: ${{ parameters.local_publishArgument }} - officialBuildProperties: ${{ parameters.local_officialBuildProperties }} + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} variables: _SignType: real - categoryName: Official buildArchitecture: x86 - publishArgument: ${{ parameters.local_publishArgument }} - officialBuildProperties: ${{ parameters.local_officialBuildProperties }} + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} variables: _SignType: real - categoryName: Official buildArchitecture: arm64 - publishArgument: ${{ parameters.local_publishArgument }} - officialBuildProperties: ${{ parameters.local_officialBuildProperties }} + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} variables: _SignType: real - ${{ else }}: @@ -183,7 +186,7 @@ extends: ############### WINDOWS ############### - - ${{ each jobParameters in parameters.local_windowsJobParameters }}: + - ${{ each jobParameters in parameters._windowsJobParameters }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: ${{ insert }}: ${{ jobParameters }} From c4afdedefe76e5876d3f0f590b03074a6e1fd204 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:55:29 -0700 Subject: [PATCH 24/90] Trying to use the matrix. --- .vsts-ci.yml | 93 +++--- .../templates/jobs/sdk-job-matrix.yml | 267 ++++++++++-------- 2 files changed, 207 insertions(+), 153 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 2f6da79d8013..c3c6b2b5d308 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,9 +26,9 @@ parameters: # default: true # ${{ else }}: # default: false -- name: _publishArgument - type: string - default: -publish +# - name: _publishArgument +# type: string +# default: -publish variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml @@ -93,39 +93,39 @@ extends: runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ### LOCAL ONLY ### # _publishArgument: -publish - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - _windowsJobParameters: - - ${{ if eq(parameters.runTests, false) }}: - - categoryName: Official - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - categoryName: Official - buildArchitecture: x86 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - categoryName: Official - buildArchitecture: arm64 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - ${{ else }}: - - categoryName: TestBuild - - categoryName: FullFramework - testFullMSBuild: true - - categoryName: TestAsTools - runTestsAsTool: true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + # _windowsJobParameters: + # - ${{ if eq(parameters.runTests, false) }}: + # - categoryName: Official + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: x86 + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: arm64 + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - ${{ else }}: + # - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true stages: @@ -185,13 +185,20 @@ extends: - ############### WINDOWS ############### - - ${{ each jobParameters in parameters._windowsJobParameters }}: - - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - ${{ insert }}: ${{ jobParameters }} - oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - runTests: ${{ parameters.runTests }} + # ############### WINDOWS ############### + # - ${{ each jobParameters in parameters._windowsJobParameters }}: + # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + # parameters: + # ${{ insert }}: ${{ jobParameters }} + # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} + # runTests: ${{ parameters.runTests }} + + + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + ${{ insert }}: ${{ jobParameters }} + oneEsPtCompat: ${{ parameters.oneEsPtCompat }} + runTests: ${{ parameters.runTests }} # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 4cacc119e088..efa1a499b724 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -23,118 +23,165 @@ parameters: # linuxContainerInfo: # ${{ if eq(parameters.runTests, false) }}: + + _publishArgument: -publish + _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + _windowsJobParameters: + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: ${{ parameters._publishArgument }} + officialBuildProperties: ${{ parameters._officialBuildProperties }} + variables: + _SignType: real + - ${{ else }}: + - categoryName: TestBuild + - categoryName: FullFramework + testFullMSBuild: true + - categoryName: TestAsTools + runTestsAsTool: true + # This uses the build step for testing, so the extra test step is not necessary. + runTests: false + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true + jobs: ############### WINDOWS ############### -- template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - helixTargetQueue: windows.amd64.vs2022.pre - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: +- ${{ each jobParameters in parameters._windowsJobParameters }}: + - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + parameters: + ${{ insert }}: ${{ jobParameters }} + oneEsPtCompat: ${{ parameters.oneEsPtCompat }} runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if eq(parameters.runTests, false) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_x86: - buildArchitecture: x86 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - ${{ else }}: - Test: {} - FullFramework: - testFullMSBuild: true - TestAsTools: - testInBuildProperties: /p:RunTestsAsTool=true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - TemplateEngine: - testName: TemplateEngineTests - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - AoT: - runAoTTests: true -############### LINUX ############### -# TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. -- template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: - runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if eq(parameters.runTests, false) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_x86: - buildArchitecture: x86 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - ${{ else }}: - Test: {} - Test_arm64: - testFullMSBuild: true -############### MACOS ############### -- template: /eng/pipelines/templates/jobs/sdk-build.yml@self - parameters: - pool: - name: Azure Pipelines - image: macOS-latest - os: macOS - helixTargetQueue: OSX.13.Amd64 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - variables: - runTests: ${{ parameters.runTests }} - # Setting this here once so it can be reused in the matrix only. - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - strategy: - matrix: - ${{ if eq(parameters.runTests, false) }}: - Official: - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) - Official_arm64: - runtimeIdentifierProperties: /p:Rid=osx-arm64 - buildArchitecture: arm64 - publishArgument: -publish - _SignType: real - officialBuildProperties: $(_officialBuildProperties) \ No newline at end of file + + +# ############### WINDOWS ############### +# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self +# parameters: +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022 +# os: windows +# helixTargetQueue: windows.amd64.vs2022.pre +# oneEsPtCompat: +# jobTemplatePath: /eng/common/templates-official/job/job.yml +# publishTaskName: 1ES.PublishBuildArtifacts@1 +# variables: +# runTests: ${{ parameters.runTests }} +# # Setting this here once so it can be reused in the matrix only. +# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) +# strategy: +# matrix: +# ${{ if eq(parameters.runTests, false) }}: +# Official: +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# Official_x86: +# buildArchitecture: x86 +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# Official_arm64: +# buildArchitecture: arm64 +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# ${{ else }}: +# Test: {} +# FullFramework: +# testFullMSBuild: true +# TestAsTools: +# testInBuildProperties: /p:RunTestsAsTool=true +# # This uses the build step for testing, so the extra test step is not necessary. +# runTests: false +# TemplateEngine: +# testName: TemplateEngineTests +# testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; +# $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj +# AoT: +# runAoTTests: true + +# ############### LINUX ############### +# # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. +# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self +# parameters: +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-ubuntu-2204 +# os: linux +# helixTargetQueue: ubuntu.2204.amd64 +# container: ubuntu2204 +# oneEsPtCompat: +# jobTemplatePath: /eng/common/templates-official/job/job.yml +# publishTaskName: 1ES.PublishBuildArtifacts@1 +# variables: +# runTests: ${{ parameters.runTests }} +# # Setting this here once so it can be reused in the matrix only. +# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) +# strategy: +# matrix: +# ${{ if eq(parameters.runTests, false) }}: +# Official: +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# Official_x86: +# buildArchitecture: x86 +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# Official_arm64: +# buildArchitecture: arm64 +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# ${{ else }}: +# Test: {} +# Test_arm64: +# testFullMSBuild: true + +# ############### MACOS ############### +# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self +# parameters: +# pool: +# name: Azure Pipelines +# image: macOS-latest +# os: macOS +# helixTargetQueue: OSX.13.Amd64 +# oneEsPtCompat: +# jobTemplatePath: /eng/common/templates-official/job/job.yml +# publishTaskName: 1ES.PublishBuildArtifacts@1 +# variables: +# runTests: ${{ parameters.runTests }} +# # Setting this here once so it can be reused in the matrix only. +# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) +# strategy: +# matrix: +# ${{ if eq(parameters.runTests, false) }}: +# Official: +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) +# Official_arm64: +# runtimeIdentifierProperties: /p:Rid=osx-arm64 +# buildArchitecture: arm64 +# publishArgument: -publish +# _SignType: real +# officialBuildProperties: $(_officialBuildProperties) \ No newline at end of file From 9b90e6ccd270b7781e8ecc08f0b3dab253c63e97 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 11:58:05 -0700 Subject: [PATCH 25/90] Forgot to remove this. --- .vsts-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index c3c6b2b5d308..d6d314bee264 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -196,7 +196,6 @@ extends: - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: - ${{ insert }}: ${{ jobParameters }} oneEsPtCompat: ${{ parameters.oneEsPtCompat }} runTests: ${{ parameters.runTests }} From 5b0ebdc54ccd495c67c5a6289d8771b8bf9a4953 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 12:00:40 -0700 Subject: [PATCH 26/90] Setting these parameters directly. --- .vsts-ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index d6d314bee264..018f37c7a4ca 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -87,10 +87,10 @@ extends: # # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ # alertWarningLevel: Critical - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + # oneEsPtCompat: + # jobTemplatePath: /eng/common/templates-official/job/job.yml + # publishTaskName: 1ES.PublishBuildArtifacts@1 + # runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ### LOCAL ONLY ### # _publishArgument: -publish # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) @@ -196,8 +196,10 @@ extends: - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: - oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - runTests: ${{ parameters.runTests }} + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self From 4746438ae7e8effafcba9f83e0c020b0cc307aba Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 12:41:15 -0700 Subject: [PATCH 27/90] Does this work? --- .vsts-ci.yml | 34 ++++++++++ .../templates/jobs/sdk-job-matrix.yml | 68 +++++++++---------- 2 files changed, 68 insertions(+), 34 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 018f37c7a4ca..d62421684341 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -200,6 +200,40 @@ extends: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + # _publishArgument: -publish + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + _windowsJobParameters: + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: -publish + officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: -publish + officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: -publish + officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + variables: + _SignType: real + - ${{ else }}: + - categoryName: TestBuild + - categoryName: FullFramework + testFullMSBuild: true + - categoryName: TestAsTools + runTestsAsTool: true + # This uses the build step for testing, so the extra test step is not necessary. + runTests: false + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index efa1a499b724..55d5d7a448b4 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -24,40 +24,40 @@ parameters: # ${{ if eq(parameters.runTests, false) }}: - _publishArgument: -publish - _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - _windowsJobParameters: - - ${{ if eq(parameters.runTests, false) }}: - - categoryName: Official - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - categoryName: Official - buildArchitecture: x86 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - categoryName: Official - buildArchitecture: arm64 - publishArgument: ${{ parameters._publishArgument }} - officialBuildProperties: ${{ parameters._officialBuildProperties }} - variables: - _SignType: real - - ${{ else }}: - - categoryName: TestBuild - - categoryName: FullFramework - testFullMSBuild: true - - categoryName: TestAsTools - runTestsAsTool: true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true + # _publishArgument: -publish + # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + # _windowsJobParameters: + # - ${{ if eq(parameters.runTests, false) }}: + # - categoryName: Official + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: x86 + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: arm64 + # publishArgument: ${{ parameters._publishArgument }} + # officialBuildProperties: ${{ parameters._officialBuildProperties }} + # variables: + # _SignType: real + # - ${{ else }}: + # - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true jobs: ############### WINDOWS ############### From 03aaf13ad8c8c75134fb4070363a3ae5720124bf Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 12:42:30 -0700 Subject: [PATCH 28/90] No default params. --- eng/pipelines/templates/jobs/sdk-job-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 55d5d7a448b4..9fec5eb00db0 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -1,4 +1,4 @@ -parameters: +# parameters: # jobTemplatePath: /eng/common/templates/job/job.yml # publishTaskName: PublishBuildArtifacts@1 # buildCategory: Build From e26bd78d7a1f6161a5dddb7f4c3635c7d97d2cf4 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 13:46:10 -0700 Subject: [PATCH 29/90] Set it up so that job-matrix is only for passing the parameters required to do the foreach logic. --- .vsts-ci.yml | 27 ++++++++++++++----- .../templates/jobs/sdk-job-matrix.yml | 19 ++++++++++--- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index d62421684341..3ef58f301db7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -46,6 +46,11 @@ variables: # TODO: What is this for? - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - group: DotNet-CLI-SDLValidation-Params +### LOCAL ONLY ### +- name: _publishArgument + value: -publish +- name: _officialBuildProperties + value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) resources: repositories: @@ -196,29 +201,37 @@ extends: - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + helixTargetQueue: windows.amd64.vs2022.pre oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) # _publishArgument: -publish # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - _windowsJobParameters: + jobParameterSets: - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official - publishArgument: -publish - officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false variables: _SignType: real - categoryName: Official buildArchitecture: x86 - publishArgument: -publish - officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false variables: _SignType: real - categoryName: Official buildArchitecture: arm64 - publishArgument: -publish - officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false variables: _SignType: real - ${{ else }}: diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 9fec5eb00db0..6c775c295877 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -59,14 +59,25 @@ # - categoryName: AoT # runAoTTests: true +parameters: + jobParameterSets: [] + jobs: -############### WINDOWS ############### -- ${{ each jobParameters in parameters._windowsJobParameters }}: +- ${{ each jobParameters in parameters.jobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml@self parameters: + ${{ insert }}: ${{ parameters }} ${{ insert }}: ${{ jobParameters }} - oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - runTests: ${{ parameters.runTests }} + + + +############### WINDOWS ############### +# - ${{ each jobParameters in parameters._windowsJobParameters }}: +# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self +# parameters: +# ${{ insert }}: ${{ jobParameters }} +# oneEsPtCompat: ${{ parameters.oneEsPtCompat }} +# runTests: ${{ parameters.runTests }} From db296fd0931305688d7894cb29e47944f67dd959 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 13:48:17 -0700 Subject: [PATCH 30/90] Needed empty default variable set. --- eng/pipelines/templates/jobs/sdk-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index b38bc148a7fb..76ac37b0f7e2 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,4 +1,5 @@ parameters: + variables: [] oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 From e2f9740a3fa26f7094b7f996e8edfb9eb92f4d4e Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 13:51:24 -0700 Subject: [PATCH 31/90] Changed it to assignment. --- eng/pipelines/templates/jobs/sdk-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 76ac37b0f7e2..e05f7810a351 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -36,8 +36,7 @@ jobs: enablePublishBuildAssets: true enableTelemetry: true enablePublishUsingPipelines: true - variables: - - ${{ insert }}: ${{ parameters.variables }} + variables: ${{ parameters.variables }} steps: ############## PREP ############### From 3bd0000e11130b192638f25f306acc367b049593 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 14:49:05 -0700 Subject: [PATCH 32/90] Trying basic Linux build. Added Linux os/arch specific scripts. --- .vsts-ci.yml | 64 ++++++++++++++++++---- eng/pipelines/templates/jobs/sdk-build.yml | 23 +++++--- 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 3ef58f301db7..2ea1998c3b7b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -191,14 +191,6 @@ extends: # ############### WINDOWS ############### - # - ${{ each jobParameters in parameters._windowsJobParameters }}: - # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - # parameters: - # ${{ insert }}: ${{ jobParameters }} - # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - # runTests: ${{ parameters.runTests }} - - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: pool: @@ -210,9 +202,8 @@ extends: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - # _publishArgument: -publish - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) jobParameterSets: + ### OFFICIAL ### - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official publishArgument: $(_publishArgument) @@ -234,6 +225,7 @@ extends: runTests: false variables: _SignType: real + ### TESTS ### - ${{ else }}: - categoryName: TestBuild - categoryName: FullFramework @@ -298,7 +290,6 @@ extends: - # ############### LINUX ############### # # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self # parameters: @@ -337,6 +328,57 @@ extends: # Test_arm64: # testFullMSBuild: true + ############### LINUX ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + jobParameterSets: + ### OFFICIAL ### + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + ### TESTS ### + # - ${{ else }}: + # - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true + # ############### MACOS ############### # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self # parameters: diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index e05f7810a351..14b5bb796a31 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -73,15 +73,20 @@ jobs: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: - - script: eng/common/build.sh - -restore -build -sign -pack -ci - -configuration $(buildConfiguration) - ${{ parameters.publishArgument }} - /p:Architecture=${{ parameters.buildArchitecture }} - /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} - ${{ parameters.runtimeSourceProperties }} - ${{ parameters.runtimeIdentifierProperties }} - ${{ parameters.officialBuildProperties }} + - script: | + source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh + source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh + initDistroRidGlobal "$os" "$arch" "" + + . "$(Build.SourcesDirectory)/eng/common/build.sh" \ + -restore -build -sign -pack -ci \ + -configuration $(buildConfiguration) \ + ${{ parameters.publishArgument }} \ + /p:Architecture=${{ parameters.buildArchitecture }} \ + /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ + ${{ parameters.runtimeSourceProperties }} \ + ${{ parameters.runtimeIdentifierProperties }} \ + ${{ parameters.officialBuildProperties }} \ /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: From 4c3a078e5539215153d16848d4cf60398b241f01 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 14:50:41 -0700 Subject: [PATCH 33/90] Wrong indentation. --- .vsts-ci.yml | 100 +++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 2ea1998c3b7b..724fd641a53e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -328,56 +328,56 @@ extends: # Test_arm64: # testFullMSBuild: true - ############### LINUX ############### - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - jobParameterSets: - ### OFFICIAL ### - - ${{ if eq(parameters.runTests, false) }}: - - categoryName: Official - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: x86 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: arm64 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - ### TESTS ### - # - ${{ else }}: - # - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true + ############### LINUX ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + jobParameterSets: + ### OFFICIAL ### + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + ### TESTS ### + # - ${{ else }}: + # - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true # ############### MACOS ############### # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self From 758ce4b208f02c6edd01b703d72a45a5d4d021fe Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 14:55:04 -0700 Subject: [PATCH 34/90] Still had tabbing wrong. --- .vsts-ci.yml | 145 ++++++++++++++++++--------------------------------- 1 file changed, 51 insertions(+), 94 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 724fd641a53e..056991722098 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -240,50 +240,56 @@ extends: - categoryName: AoT runAoTTests: true - - # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - # parameters: - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-windows-2022 - # os: windows - # helixTargetQueue: windows.amd64.vs2022.pre - # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - # # variables: - # # runTests: ${{ parameters.runTests }} - # # # Setting this here once so it can be reused in the matrix only. - # # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - # strategy: - # matrix: - # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Official: - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_x86: - # buildArchitecture: x86 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_arm64: - # buildArchitecture: arm64 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # ${{ else }}: - # Test: {} - # FullFramework: - # testFullMSBuild: true - # TestAsTools: - # testInBuildProperties: /p:RunTestsAsTool=true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # TemplateEngine: - # testName: TemplateEngineTests - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # AoT: - # runAoTTests: true + ############### LINUX ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + jobParameterSets: + ### OFFICIAL ### + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + ### TESTS ### + # - ${{ else }}: + # - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true @@ -328,56 +334,7 @@ extends: # Test_arm64: # testFullMSBuild: true - ############### LINUX ############### - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - jobParameterSets: - ### OFFICIAL ### - - ${{ if eq(parameters.runTests, false) }}: - - categoryName: Official - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: x86 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: arm64 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - ### TESTS ### - # - ${{ else }}: - # - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true + # ############### MACOS ############### # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self From 8381441e92483b65bd56733df02eabda5e2329c4 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 14:59:00 -0700 Subject: [PATCH 35/90] Does this work? --- .vsts-ci.yml | 26 +++++++++++----------- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 056991722098..7e03c9b7387e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -277,19 +277,19 @@ extends: variables: _SignType: real ### TESTS ### - # - ${{ else }}: - # - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true + - ${{ else }}: + - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 14b5bb796a31..fcccfead7b60 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -73,7 +73,7 @@ jobs: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: - - script: | + - script: > source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh initDistroRidGlobal "$os" "$arch" "" From 0498c933dd123161bb48f6213f3722320880cce2 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:00:46 -0700 Subject: [PATCH 36/90] Is this the problem? --- eng/pipelines/templates/jobs/sdk-build.yml | 39 +++++++++++++--------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index fcccfead7b60..505cc906bdcc 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -73,21 +73,30 @@ jobs: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: - - script: > - source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh - source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh - initDistroRidGlobal "$os" "$arch" "" - - . "$(Build.SourcesDirectory)/eng/common/build.sh" \ - -restore -build -sign -pack -ci \ - -configuration $(buildConfiguration) \ - ${{ parameters.publishArgument }} \ - /p:Architecture=${{ parameters.buildArchitecture }} \ - /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ - ${{ parameters.runtimeSourceProperties }} \ - ${{ parameters.runtimeIdentifierProperties }} \ - ${{ parameters.officialBuildProperties }} \ - /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + - script: eng/common/build.sh + -restore -build -sign -pack -ci + -configuration $(buildConfiguration) + ${{ parameters.publishArgument }} + /p:Architecture=${{ parameters.buildArchitecture }} + /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} + ${{ parameters.runtimeSourceProperties }} + ${{ parameters.runtimeIdentifierProperties }} + ${{ parameters.officialBuildProperties }} + # - script: > + # source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh + # source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh + # initDistroRidGlobal "$os" "$arch" "" + + # . "$(Build.SourcesDirectory)/eng/common/build.sh" \ + # -restore -build -sign -pack -ci \ + # -configuration $(buildConfiguration) \ + # ${{ parameters.publishArgument }} \ + # /p:Architecture=${{ parameters.buildArchitecture }} \ + # /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ + # ${{ parameters.runtimeSourceProperties }} \ + # ${{ parameters.runtimeIdentifierProperties }} \ + # ${{ parameters.officialBuildProperties }} \ + # /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: # Required by Arcade for passing to Helix From f1b94eecf53ddd3ef9db80840bccf79c6d7bf2f6 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:01:31 -0700 Subject: [PATCH 37/90] Comment this section out. --- .vsts-ci.yml | 98 ++++++++++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7e03c9b7387e..bce0e5dd2742 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -241,55 +241,55 @@ extends: runAoTTests: true ############### LINUX ############### - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self - parameters: - pool: - name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204 - os: linux - helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 - oneEsPtCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml - publishTaskName: 1ES.PublishBuildArtifacts@1 - runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - jobParameterSets: - ### OFFICIAL ### - - ${{ if eq(parameters.runTests, false) }}: - - categoryName: Official - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: x86 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - - categoryName: Official - buildArchitecture: arm64 - publishArgument: $(_publishArgument) - officialBuildProperties: $(_officialBuildProperties) - runTests: false - variables: - _SignType: real - ### TESTS ### - - ${{ else }}: - - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true + # - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + # parameters: + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-ubuntu-2204 + # os: linux + # helixTargetQueue: ubuntu.2204.amd64 + # container: ubuntu2204 + # oneEsPtCompat: + # jobTemplatePath: /eng/common/templates-official/job/job.yml + # publishTaskName: 1ES.PublishBuildArtifacts@1 + # runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + # jobParameterSets: + # ### OFFICIAL ### + # - ${{ if eq(parameters.runTests, false) }}: + # - categoryName: Official + # publishArgument: $(_publishArgument) + # officialBuildProperties: $(_officialBuildProperties) + # runTests: false + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: x86 + # publishArgument: $(_publishArgument) + # officialBuildProperties: $(_officialBuildProperties) + # runTests: false + # variables: + # _SignType: real + # - categoryName: Official + # buildArchitecture: arm64 + # publishArgument: $(_publishArgument) + # officialBuildProperties: $(_officialBuildProperties) + # runTests: false + # variables: + # _SignType: real + # ### TESTS ### + # - ${{ else }}: + # - categoryName: TestBuild + # # - categoryName: FullFramework + # # testFullMSBuild: true + # # - categoryName: TestAsTools + # # runTestsAsTool: true + # # # This uses the build step for testing, so the extra test step is not necessary. + # # runTests: false + # # - categoryName: TemplateEngine + # # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # # - categoryName: AoT + # # runAoTTests: true From 20791c8d111262a4a489dff5728169f7d51d4463 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:03:12 -0700 Subject: [PATCH 38/90] Huh --- .vsts-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index bce0e5dd2742..8ef4957f5cd5 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -203,7 +203,6 @@ extends: publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - ### OFFICIAL ### - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official publishArgument: $(_publishArgument) @@ -225,7 +224,6 @@ extends: runTests: false variables: _SignType: real - ### TESTS ### - ${{ else }}: - categoryName: TestBuild - categoryName: FullFramework From e0b4bf01c24e7a85f1a0a01ff039f6405fb7575d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:06:27 -0700 Subject: [PATCH 39/90] Variables was wrong. --- .vsts-ci.yml | 100 +++++++++++---------- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 8ef4957f5cd5..7e03c9b7387e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -203,6 +203,7 @@ extends: publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: + ### OFFICIAL ### - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official publishArgument: $(_publishArgument) @@ -224,6 +225,7 @@ extends: runTests: false variables: _SignType: real + ### TESTS ### - ${{ else }}: - categoryName: TestBuild - categoryName: FullFramework @@ -239,55 +241,55 @@ extends: runAoTTests: true ############### LINUX ############### - # - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self - # parameters: - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-ubuntu-2204 - # os: linux - # helixTargetQueue: ubuntu.2204.amd64 - # container: ubuntu2204 - # oneEsPtCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml - # publishTaskName: 1ES.PublishBuildArtifacts@1 - # runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - # jobParameterSets: - # ### OFFICIAL ### - # - ${{ if eq(parameters.runTests, false) }}: - # - categoryName: Official - # publishArgument: $(_publishArgument) - # officialBuildProperties: $(_officialBuildProperties) - # runTests: false - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: x86 - # publishArgument: $(_publishArgument) - # officialBuildProperties: $(_officialBuildProperties) - # runTests: false - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: arm64 - # publishArgument: $(_publishArgument) - # officialBuildProperties: $(_officialBuildProperties) - # runTests: false - # variables: - # _SignType: real - # ### TESTS ### - # - ${{ else }}: - # - categoryName: TestBuild - # # - categoryName: FullFramework - # # testFullMSBuild: true - # # - categoryName: TestAsTools - # # runTestsAsTool: true - # # # This uses the build step for testing, so the extra test step is not necessary. - # # runTests: false - # # - categoryName: TemplateEngine - # # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # # - categoryName: AoT - # # runAoTTests: true + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + helixTargetQueue: ubuntu.2204.amd64 + container: ubuntu2204 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + jobParameterSets: + ### OFFICIAL ### + - ${{ if eq(parameters.runTests, false) }}: + - categoryName: Official + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: x86 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: Official + buildArchitecture: arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + ### TESTS ### + - ${{ else }}: + - categoryName: TestBuild + # - categoryName: FullFramework + # testFullMSBuild: true + # - categoryName: TestAsTools + # runTestsAsTool: true + # # This uses the build step for testing, so the extra test step is not necessary. + # runTests: false + # - categoryName: TemplateEngine + # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # - categoryName: AoT + # runAoTTests: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 505cc906bdcc..5586a4bc255a 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,5 +1,5 @@ parameters: - variables: [] + variables: {} oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 From 3955f5f8b4d1ce985c7e72b639242c44568e9c90 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:08:59 -0700 Subject: [PATCH 40/90] Does this work? --- eng/pipelines/templates/jobs/sdk-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 5586a4bc255a..0c378ec9d1ca 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,5 +1,5 @@ parameters: - variables: {} + variables: '' oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 @@ -36,7 +36,9 @@ jobs: enablePublishBuildAssets: true enableTelemetry: true enablePublishUsingPipelines: true - variables: ${{ parameters.variables }} + ${{ if ne(parameters.variables, '') }}: + variables: + ${{ insert }}: ${{ parameters.variables }} steps: ############## PREP ############### From 1ffbfe45353edeecdebf0f3485758bb6c06c1f8c Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:12:37 -0700 Subject: [PATCH 41/90] I don't understand. --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 0c378ec9d1ca..c13cab10a49a 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -38,7 +38,7 @@ jobs: enablePublishUsingPipelines: true ${{ if ne(parameters.variables, '') }}: variables: - ${{ insert }}: ${{ parameters.variables }} + - ${{ insert }}: ${{ parameters.variables }} steps: ############## PREP ############### From a86ca5fb80cb35e1361fd472add4ad0211d2f1e1 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:16:00 -0700 Subject: [PATCH 42/90] Does this work? --- eng/pipelines/templates/jobs/sdk-build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index c13cab10a49a..651b231b6264 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,5 +1,5 @@ parameters: - variables: '' + variables: [] oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 @@ -36,9 +36,8 @@ jobs: enablePublishBuildAssets: true enableTelemetry: true enablePublishUsingPipelines: true - ${{ if ne(parameters.variables, '') }}: - variables: - - ${{ insert }}: ${{ parameters.variables }} + variables: + - ${{ insert }}: ${{ parameters.variables }} steps: ############## PREP ############### From b6039ade5f773f456f62424ecec849550bc994f8 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:17:01 -0700 Subject: [PATCH 43/90] How about this? --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 651b231b6264..3f16b99fa0dc 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,5 +1,5 @@ parameters: - variables: [] + variables: {} oneEsPtCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 From b81a92f7d77774cbd34d9f5de833c8201cbd1e71 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 9 May 2024 15:22:00 -0700 Subject: [PATCH 44/90] Trying Linux build adjustments. --- eng/pipelines/templates/jobs/sdk-build.yml | 41 +++++++++------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 3f16b99fa0dc..9492acd42b88 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -48,7 +48,7 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ else }}: - - script: ./$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token + - script: . "$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh" $(Build.SourcesDirectory)/NuGet.config $Token displayName: Setup Private Feeds Credentials env: Token: $(dn-bot-dnceng-artifact-feeds-rw) @@ -74,30 +74,21 @@ jobs: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: - - script: eng/common/build.sh - -restore -build -sign -pack -ci - -configuration $(buildConfiguration) - ${{ parameters.publishArgument }} - /p:Architecture=${{ parameters.buildArchitecture }} - /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} - ${{ parameters.runtimeSourceProperties }} - ${{ parameters.runtimeIdentifierProperties }} - ${{ parameters.officialBuildProperties }} - # - script: > - # source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh - # source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh - # initDistroRidGlobal "$os" "$arch" "" - - # . "$(Build.SourcesDirectory)/eng/common/build.sh" \ - # -restore -build -sign -pack -ci \ - # -configuration $(buildConfiguration) \ - # ${{ parameters.publishArgument }} \ - # /p:Architecture=${{ parameters.buildArchitecture }} \ - # /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ - # ${{ parameters.runtimeSourceProperties }} \ - # ${{ parameters.runtimeIdentifierProperties }} \ - # ${{ parameters.officialBuildProperties }} \ - # /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + - script: | + source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh + source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh + initDistroRidGlobal "$os" "$arch" "" + + . "$(Build.SourcesDirectory)/eng/common/build.sh" \ + -restore -build -sign -pack -ci \ + -configuration $(buildConfiguration) \ + ${{ parameters.publishArgument }} \ + /p:Architecture=${{ parameters.buildArchitecture }} \ + /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ + ${{ parameters.runtimeSourceProperties }} \ + ${{ parameters.runtimeIdentifierProperties }} \ + ${{ parameters.officialBuildProperties }} \ + /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: # Required by Arcade for passing to Helix From 73b97c28e4156cd1b5211bef6c2a332843b20921 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 10 May 2024 09:43:31 -0700 Subject: [PATCH 45/90] Trying to fix using Agent.JobName since it is bugged and uses the display name of the job. --- eng/Publishing.props | 4 ++-- eng/pipelines/templates/jobs/sdk-build.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 0c8db5490de9..1d781e4c3350 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -146,9 +146,9 @@ --> $(AGENT_OS) $(OS) - $(AssetManifestOS)-$(AGENT_JOBNAME) + $(AssetManifestOS)-$(SYSTEM_PHASENAME) $(AssetManifestOS) - $(AssetManifestOS)-$(Architecture) + $(AssetManifestOS)-$(Architecture) $(BaseAssetManifestFileName)-installers $(ArtifactsLogDir)AssetManifest\$(InstallersAssetManifestFileName).xml diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 9492acd42b88..2df8bac3ff5b 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -41,6 +41,8 @@ jobs: steps: ############## PREP ############### + - powershell: 'Get-ChildItem env:' + displayName: List Environment Variables - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token From 2b24aa14e3f2791d6c392026aa9a000a0265f766 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 10 May 2024 10:26:46 -0700 Subject: [PATCH 46/90] Added signArgument so that it is only used for windows. Checking if Helix info is needed for non-tests. --- .vsts-ci.yml | 5 ++++ eng/Publishing.props | 2 +- eng/pipelines/templates/jobs/sdk-build.yml | 25 ++++++++----------- .../templates/variables/sdk-defaults.yml | 1 + 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7e03c9b7387e..d87524232bf9 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -49,6 +49,8 @@ variables: ### LOCAL ONLY ### - name: _publishArgument value: -publish +- name: _signArgument + value: -sign /p:SignCoreSdk=true - name: _officialBuildProperties value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) @@ -207,6 +209,7 @@ extends: - ${{ if eq(parameters.runTests, false) }}: - categoryName: Official publishArgument: $(_publishArgument) + signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false variables: @@ -214,6 +217,7 @@ extends: - categoryName: Official buildArchitecture: x86 publishArgument: $(_publishArgument) + signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false variables: @@ -221,6 +225,7 @@ extends: - categoryName: Official buildArchitecture: arm64 publishArgument: $(_publishArgument) + signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false variables: diff --git a/eng/Publishing.props b/eng/Publishing.props index 1d781e4c3350..5ec015e340b4 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -142,7 +142,7 @@ $(AGENT_OS) $(OS) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 2df8bac3ff5b..da5acca4b35e 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -13,6 +13,7 @@ parameters: runAoTTests: false ### MSBUILD ### publishArgument: '' + signArgument: '' runTestsAsTool: false runtimeSourceProperties: '' runtimeIdentifierProperties: '' @@ -41,8 +42,6 @@ jobs: steps: ############## PREP ############### - - powershell: 'Get-ChildItem env:' - displayName: List Environment Variables - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token @@ -58,9 +57,10 @@ jobs: ############### BUILDING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 - -restore -build -sign -pack -ci -nativeToolsOnMachine + -restore -build -pack -ci -nativeToolsOnMachine -configuration $(buildConfiguration) ${{ parameters.publishArgument }} + ${{ parameters.signArgument }} /p:Architecture=${{ parameters.buildArchitecture }} /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} ${{ parameters.runtimeSourceProperties }} @@ -70,9 +70,8 @@ jobs: displayName: Build env: # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) + # SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: @@ -82,9 +81,10 @@ jobs: initDistroRidGlobal "$os" "$arch" "" . "$(Build.SourcesDirectory)/eng/common/build.sh" \ - -restore -build -sign -pack -ci \ + -restore -build -pack -ci \ -configuration $(buildConfiguration) \ ${{ parameters.publishArgument }} \ + ${{ parameters.signArgument }} \ /p:Architecture=${{ parameters.buildArchitecture }} \ /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ ${{ parameters.runtimeSourceProperties }} \ @@ -94,9 +94,8 @@ jobs: displayName: Build env: # Required by Arcade for passing to Helix - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally - HelixAccessToken: $(HelixApiAccessToken) + # SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} @@ -116,9 +115,8 @@ jobs: displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() env: - # Required by Arcade for passing to Helix + # Required by Arcade for running in Helix. SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) RunAoTTests: ${{ parameters.runAoTTests }} - ${{ else }}: @@ -134,9 +132,8 @@ jobs: displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() env: - # Required by Arcade for passing to Helix + # Required by Arcade for running in Helix. SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # Only set when ran internally HelixAccessToken: $(HelixApiAccessToken) RunAoTTests: ${{ parameters.runAoTTests }} diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 708b91503e02..5936171d7c9f 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -23,6 +23,7 @@ variables: # Both this (used in Arcade for the MicroBuildSigningPlugin) and DotNetSignType (used in Arcade in Sign.proj) are necessary to set the sign type. _SignType: test _TeamName: DotNetCore + # This ends up being set by the DotNet-HelixApi-Access variable group when running internally. HelixApiAccessToken: '' ############### OTHER ############### From c9eafc34a352d01e63f5ed1ad24d4003262469a1 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 10 May 2024 12:11:12 -0700 Subject: [PATCH 47/90] Added the rest of the CI windows and linux build jobs. --- .vsts-ci.yml | 117 ++++++++++++++++++--- eng/pipelines/templates/jobs/sdk-build.yml | 16 +-- 2 files changed, 114 insertions(+), 19 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index d87524232bf9..e1463ae17df4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -53,6 +53,8 @@ variables: value: -sign /p:SignCoreSdk=true - name: _officialBuildProperties value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) +- name: _linuxOsProperties + value: /p:OSName=linux /p:IsLinuxPortable=true resources: repositories: @@ -205,10 +207,10 @@ extends: publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - ### OFFICIAL ### - ${{ if eq(parameters.runTests, false) }}: + ### OFFICIAL ### - categoryName: Official - publishArgument: $(_publishArgument) + publishArgument: $(_publishArgument) /p:PublishInternalAsset=true signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false @@ -230,8 +232,35 @@ extends: runTests: false variables: _SignType: real - ### TESTS ### + ### PGO ### + - categoryName: PGO + pgoInstrument: true + publishArgument: $(_publishArgument) /p:PublishInternalAsset=true + signArgument: $(_signArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: PGO + pgoInstrument: true + buildArchitecture: x86 + publishArgument: $(_publishArgument) + signArgument: $(_signArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real + - categoryName: PGO + pgoInstrument: true + buildArchitecture: arm64 + publishArgument: $(_publishArgument) + signArgument: $(_signArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + variables: + _SignType: real - ${{ else }}: + ### TESTS ### - categoryName: TestBuild - categoryName: FullFramework testFullMSBuild: true @@ -253,36 +282,98 @@ extends: image: 1es-ubuntu-2204 os: linux helixTargetQueue: ubuntu.2204.amd64 - container: ubuntu2204 oneEsPtCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - ### OFFICIAL ### - ${{ if eq(parameters.runTests, false) }}: + ### OFFICIAL ### + # Note: These builds are also portable like the Portable category, but that category uses containers, and don't publish zips and tarballs. - categoryName: Official publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) runTests: false - variables: - _SignType: real - categoryName: Official - buildArchitecture: x86 + buildArchitecture: arm + runtimeIdentifier: linux-arm publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) runTests: false - variables: - _SignType: real - categoryName: Official buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) + runTests: false + ### PORTABLE ### + - categoryName: Portable + container: ubuntu2204DebPkg + # Do not publish zips and tarballs. The linux-x64 binaries are already published by Official. + publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false + officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) /p:BuildSdkDeb=true + runTests: false + - categoryName: Portable + container: cblMariner20Fpm + # Do not publish zips and tarballs. The linux-x64 binaries are already published by Official. + publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false + officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) /p:IsRPMBasedDistro=true + runTests: false + - categoryName: Portable + container: cblMariner20Fpm + buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 + # Do not publish zips and tarballs. The linux-arm64 binaries are already published by Official. + publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true + officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) /p:IsRPMBasedDistro=true + runTests: false + ### MUSL ### + - categoryName: Musl + container: alpine319WithNode + runtimeIdentifier: linux-musl-x64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + # Pass in HostOSName when running on alpine. + osProperties: /p:HostOSName=linux-musl + runTests: false + - categoryName: Musl + container: mariner20CrossArm + buildArchitecture: arm + runtimeIdentifier: linux-musl-arm + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + osProperties: /p:OSName=linux-musl + runTests: false + - categoryName: Musl + buildArchitecture: arm64 + runtimeIdentifier: linux-musl-arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + osProperties: /p:OSName=linux-musl + runTests: false + ### PGO ### + - categoryName: PGO + pgoInstrument: true + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) + runTests: false + - categoryName: PGO + pgoInstrument: true + buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + osProperties: $(_linuxOsProperties) runTests: false - variables: - _SignType: real - ### TESTS ### - ${{ else }}: + ### TESTS ### - categoryName: TestBuild # - categoryName: FullFramework # testFullMSBuild: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index da5acca4b35e..af9822ca3fe3 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -5,18 +5,20 @@ parameters: publishTaskName: PublishBuildArtifacts@1 container: '' categoryName: Build - buildArchitecture: x64 runTests: true testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj ### ENV VARS ### testFullMSBuild: false runAoTTests: false ### MSBUILD ### + buildArchitecture: x64 publishArgument: '' signArgument: '' runTestsAsTool: false + pgoInstrument: false + runtimeIdentifier: linux-x64 + osProperties: '' runtimeSourceProperties: '' - runtimeIdentifierProperties: '' officialBuildProperties: '' jobs: @@ -63,8 +65,8 @@ jobs: ${{ parameters.signArgument }} /p:Architecture=${{ parameters.buildArchitecture }} /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} + /p:PgoInstrument=${{ parameters.pgoInstrument }} ${{ parameters.runtimeSourceProperties }} - ${{ parameters.runtimeIdentifierProperties }} ${{ parameters.officialBuildProperties }} /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build @@ -87,8 +89,10 @@ jobs: ${{ parameters.signArgument }} \ /p:Architecture=${{ parameters.buildArchitecture }} \ /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ + /p:PgoInstrument=${{ parameters.pgoInstrument }} \ + /p:Rid=${{ parameters.runtimeIdentifier }} \ + ${{ parameters.osProperties }} \ ${{ parameters.runtimeSourceProperties }} \ - ${{ parameters.runtimeIdentifierProperties }} \ ${{ parameters.officialBuildProperties }} \ /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build @@ -109,7 +113,6 @@ jobs: -projects ${{ parameters.testProjects }} /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} - ${{ parameters.runtimeIdentifierProperties }} /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests @@ -125,8 +128,9 @@ jobs: -configuration $(buildConfiguration) -projects ${{ parameters.testProjects }} /p:Architecture=${{ parameters.buildArchitecture }} + /p:Rid=${{ parameters.runtimeIdentifier }} + ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - ${{ parameters.runtimeIdentifierProperties }} /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests From 51c4d0cbf2939450f0ba531eb3ef7c79fc58b9cb Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 10 May 2024 14:23:37 -0700 Subject: [PATCH 48/90] Added the rest of the jobs for Mac and the linux test jobs. --- .vsts-ci.yml | 252 ++++++--------------- eng/pipelines/templates/jobs/sdk-build.yml | 8 +- 2 files changed, 72 insertions(+), 188 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e1463ae17df4..ba5a53a9fda4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -20,9 +20,9 @@ parameters: - name: runTests displayName: Run Tests type: boolean - default: false + default: ${{ eq(variables['Build.Reason'], 'PullRequest') }} # 💢 "A template expression is not allowed in this context" - # ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: # default: true # ${{ else }}: # default: false @@ -96,46 +96,6 @@ extends: # # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ # alertWarningLevel: Critical - # oneEsPtCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml - # publishTaskName: 1ES.PublishBuildArtifacts@1 - # runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - ### LOCAL ONLY ### - # _publishArgument: -publish - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - # _windowsJobParameters: - # - ${{ if eq(parameters.runTests, false) }}: - # - categoryName: Official - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: x86 - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: arm64 - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - ${{ else }}: - # - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true - stages: - stage: build @@ -153,47 +113,6 @@ extends: # targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration # artifactName: buildConfiguration - - # - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self - # parameters: - # oneEsPtCompat: ${{ parameters.oneEsPtCompat }} - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-windows-2022 - # os: windows - # helixTargetQueue: windows.amd64.vs2022.pre - # ${{ if eq(parameters.runTests, false) }}: - # Official: - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_x86: - # buildArchitecture: x86 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_arm64: - # buildArchitecture: arm64 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # ${{ else }}: - # Test: {} - # FullFramework: - # testFullMSBuild: true - # TestAsTools: - # testInBuildProperties: /p:RunTestsAsTool=true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # TemplateEngine: - # testName: TemplateEngineTests - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # AoT: - # runAoTTests: true - - - # ############### WINDOWS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: @@ -289,7 +208,7 @@ extends: jobParameterSets: - ${{ if eq(parameters.runTests, false) }}: ### OFFICIAL ### - # Note: These builds are also portable like the Portable category, but that category uses containers, and don't publish zips and tarballs. + # Note: These builds are also portable like the Portable category, but that category uses containers, and doesn't publish zips and tarballs. - categoryName: Official publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) @@ -339,7 +258,7 @@ extends: runtimeIdentifier: linux-musl-x64 publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - # Pass in HostOSName when running on alpine. + # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl runTests: false - categoryName: Musl @@ -375,108 +294,69 @@ extends: - ${{ else }}: ### TESTS ### - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true - - - - - - - # # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. - # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - # parameters: - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-ubuntu-2204 - # os: linux - # helixTargetQueue: ubuntu.2204.amd64 - # container: ubuntu2204 - # oneEsPtCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml - # publishTaskName: 1ES.PublishBuildArtifacts@1 - # variables: - # runTests: ${{ parameters.runTests }} - # # Setting this here once so it can be reused in the matrix only. - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - # strategy: - # matrix: - # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Official: - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_x86: - # buildArchitecture: x86 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_arm64: - # buildArchitecture: arm64 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # ${{ else }}: - # Test: {} - # Test_arm64: - # testFullMSBuild: true - - - - # ############### MACOS ############### - # - template: /eng/pipelines/templates/jobs/sdk-build.yml@self - # parameters: - # pool: - # name: Azure Pipelines - # image: macOS-latest - # os: macOS - # helixTargetQueue: OSX.13.Amd64 - # oneEsPtCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml - # publishTaskName: 1ES.PublishBuildArtifacts@1 - # variables: - # runTests: ${{ parameters.runTests }} - # # Setting this here once so it can be reused in the matrix only. - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - # strategy: - # matrix: - # ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Official: - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - # Official_arm64: - # runtimeIdentifierProperties: /p:Rid=osx-arm64 - # buildArchitecture: arm64 - # publishArgument: -publish - # _SignType: real - # officialBuildProperties: $(_officialBuildProperties) - - - - - - - - - - - - - - - - + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 + osProperties: $(_linuxOsProperties) + # Don't run the tests on arm64. Only perform the build itself. + runTests: false + - categoryName: TestBuild + container: ubuntu2204 + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + container: fedora39 + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + container: centosStream9 + osProperties: /p:OSName=linux + - categoryName: TestBuild + container: debian11Amd64 + osProperties: /p:OSName=linux /p:BuildSdkDeb=true + - categoryName: TestBuild + container: alpine319WithNode + runtimeIdentifier: linux-musl-x64 + # Use HostOSName when running on alpine. + osProperties: /p:HostOSName=linux-musl + - categoryName: TemplateEngine + osProperties: $(_linuxOsProperties) + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + ############### MACOS ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: Azure Pipelines + image: macOS-latest + os: macOS + helixTargetQueue: OSX.13.Amd64 + oneEsPtCompat: + jobTemplatePath: /eng/common/templates-official/job/job.yml + publishTaskName: 1ES.PublishBuildArtifacts@1 + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + jobParameterSets: + - ${{ if eq(parameters.runTests, false) }}: + ### OFFICIAL ### + - categoryName: Official + runtimeIdentifier: osx-x64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + - categoryName: Official + buildArchitecture: arm64 + runtimeIdentifier: osx-arm64 + publishArgument: $(_publishArgument) + officialBuildProperties: $(_officialBuildProperties) + runTests: false + - ${{ else }}: + ### TESTS ### + - categoryName: TestBuild + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; + $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true #################### SOURCE BUILD #################### diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index af9822ca3fe3..55b26a9a98b5 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -50,8 +50,9 @@ jobs: displayName: Setup Private Feeds Credentials env: Token: $(dn-bot-dnceng-artifact-feeds-rw) + # The PowerShell script above would work fine on Linux/Mac, but the Linux containers don't have PowerShell installed. - ${{ else }}: - - script: . "$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh" $(Build.SourcesDirectory)/NuGet.config $Token + - script: . $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token displayName: Setup Private Feeds Credentials env: Token: $(dn-bot-dnceng-artifact-feeds-rw) @@ -82,7 +83,7 @@ jobs: source $(Build.SourcesDirectory)/eng/common/native/init-distro-rid.sh initDistroRidGlobal "$os" "$arch" "" - . "$(Build.SourcesDirectory)/eng/common/build.sh" \ + . $(Build.SourcesDirectory)/eng/common/build.sh \ -restore -build -pack -ci \ -configuration $(buildConfiguration) \ ${{ parameters.publishArgument }} \ @@ -105,6 +106,9 @@ jobs: ############### TESTING ############### - ${{ if eq(parameters.runTests, true) }}: + - ${{ if eq(parameters.runAoTTests, true) }}: + - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update + displayName: Install wasm-tools Workload - ${{ if eq(parameters.pool.os, 'windows') }}: # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 From 661b380ce66a0ffc7250f627a09a09358b0fe49e Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 10 May 2024 14:40:49 -0700 Subject: [PATCH 49/90] Just set it to false. --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index ba5a53a9fda4..6f141e791f1b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -20,7 +20,7 @@ parameters: - name: runTests displayName: Run Tests type: boolean - default: ${{ eq(variables['Build.Reason'], 'PullRequest') }} + default: false # 💢 "A template expression is not allowed in this context" # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: # default: true From 149e8d3b8d91922c533fee9df6382a0958dc17cb Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:22:22 -0700 Subject: [PATCH 50/90] Enabled various other pipeline jobs/stages that were previously running. Allowed a way for OneLocBuild branch targeting to be specified. General formatting and cleanup. --- .vsts-ci.yml | 131 ++++++----- eng/Publishing.props | 10 +- .../dotnet-format-integration.yml | 151 ++++++------ eng/pipelines/templates/jobs/sdk-build.yml | 59 +---- .../templates/jobs/sdk-job-matrix.yml | 215 ++---------------- 5 files changed, 175 insertions(+), 391 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6f141e791f1b..e1796208fbc1 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,9 +26,6 @@ parameters: # default: true # ${{ else }}: # default: false -# - name: _publishArgument -# type: string -# default: -publish variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml @@ -43,7 +40,7 @@ variables: - ${{ if and(eq(parameters.runTests, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - name: PostBuildSign value: false -# TODO: What is this for? +# Provides TSA variables for automatic bug reporting. - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - group: DotNet-CLI-SDLValidation-Params ### LOCAL ONLY ### @@ -91,29 +88,17 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - # componentgovernance: - # # Setting to Critical as aspnet templates currently cause a High alert (default alert level is High). - # # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 - # # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - # alertWarningLevel: Critical + componentgovernance: + # Setting to Critical as aspnet templates currently cause a High alert (default alert level is High). + # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 + # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ + alertWarningLevel: Critical stages: - stage: build displayName: Build jobs: - # - job: Publish_Build_Configuration - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-windows-2022 - # os: windows - # steps: - # - task: 1ES.PublishPipelineArtifact@1 - # displayName: Publish Build Config - # inputs: - # targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration - # artifactName: buildConfiguration - - # ############### WINDOWS ############### + ############### WINDOWS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: pool: @@ -121,12 +106,14 @@ extends: image: 1es-windows-2022 os: windows helixTargetQueue: windows.amd64.vs2022.pre - oneEsPtCompat: + oneESCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + publishRetryConfig: true + locBranch: release/8.0.4xx jobParameterSets: - - ${{ if eq(parameters.runTests, false) }}: + - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: ### OFFICIAL ### - categoryName: Official publishArgument: $(_publishArgument) /p:PublishInternalAsset=true @@ -201,12 +188,12 @@ extends: image: 1es-ubuntu-2204 os: linux helixTargetQueue: ubuntu.2204.amd64 - oneEsPtCompat: + oneESCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - - ${{ if eq(parameters.runTests, false) }}: + - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: ### OFFICIAL ### # Note: These builds are also portable like the Portable category, but that category uses containers, and doesn't publish zips and tarballs. - categoryName: Official @@ -331,12 +318,12 @@ extends: image: macOS-latest os: macOS helixTargetQueue: OSX.13.Amd64 - oneEsPtCompat: + oneESCompat: jobTemplatePath: /eng/common/templates-official/job/job.yml publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - - ${{ if eq(parameters.runTests, false) }}: + - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: ### OFFICIAL ### - categoryName: Official runtimeIdentifier: osx-x64 @@ -359,24 +346,19 @@ extends: runAoTTests: true - #################### SOURCE BUILD #################### - - # - template: /eng/common/templates-official/job/source-build.yml@self - # parameters: - # platform: - # name: Managed - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # # dotnet-format builds - # - template: /eng/dotnet-format/dotnet-format-integration.yml@self - # parameters: - # PublishTaskName: 1ES.PublishBuildArtifacts@1 - - - - + ############### SOURCE BUILD ############### + - template: /eng/common/templates-official/job/source-build.yml@self + parameters: + platform: + name: Managed + container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + ############### DOTNET-FORMAT ############### + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/dotnet-format/dotnet-format-integration.yml@self + parameters: + oneESCompat: + publishTaskName: 1ES.PublishBuildArtifacts@1 # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: @@ -392,25 +374,40 @@ extends: # image: 1es-windows-2022 # os: windows - # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - # - template: /eng/common/templates-official/post-build/post-build.yml@self - # parameters: - # publishingInfraVersion: 3 - # enableSymbolValidation: false - # enableSigningValidation: false - # enableNugetValidation: false - # enableSourceLinkValidation: false - # publishInstallersAndChecksums: true - # publishAssetsImmediately: true - # SDLValidationParameters: - # enable: false - # params: ' -SourceToolsList @("policheck","credscan") - # -TsaInstanceURL $(_TsaInstanceURL) - # -TsaProjectName $(_TsaProjectName) - # -TsaNotificationEmail $(_TsaNotificationEmail) - # -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - # -TsaBugAreaPath $(_TsaBugAreaPath) - # -TsaIterationPath $(_TsaIterationPath) - # -TsaRepositoryName "dotnet-sdk" - # -TsaCodebaseName "dotnet-sdk" - # -TsaPublish $True' + + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - stage: publish + displayName: Publish + dependsOn: build + jobs: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + enableSourceLinkValidation: false + publishInstallersAndChecksums: true + publishAssetsImmediately: true + SDLValidationParameters: + enable: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "dotnet-sdk" + -TsaCodebaseName "dotnet-sdk" + -TsaPublish $True' diff --git a/eng/Publishing.props b/eng/Publishing.props index 5ec015e340b4..e5a7e8b47f7e 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -71,13 +71,13 @@ - Shipping - Release + Shipping + Release - - - + + + diff --git a/eng/dotnet-format/dotnet-format-integration.yml b/eng/dotnet-format/dotnet-format-integration.yml index 8e00c7b45397..bff84d7c707e 100644 --- a/eng/dotnet-format/dotnet-format-integration.yml +++ b/eng/dotnet-format/dotnet-format-integration.yml @@ -1,61 +1,65 @@ parameters: - - name: PublishTaskName - default: PublishBuildArtifacts@1 - - name: TestArguments - type: object - default: - - Name: Roslyn - _repo: "https://github.com/dotnet/roslyn" - _repoName: "dotnet/roslyn" - _targetSolution: "Compilers.slnf" - _branchName: "main" - _sha: "a3bb37003aeccad012a6e7dd220977599e8b8e65" - _useParentSdk: 0 - - Name: sdk - _repo: "https://github.com/dotnet/sdk" - _repoName: "dotnet/sdk" - _targetSolution: "sdk.sln" - _branchName: "main" - _sha: "be25db95c376bffd508a023399ddd34392fe6458" - _useParentSdk: 0 - - Name: project_system - _repo: "https://github.com/dotnet/project-system" - _repoName: "dotnet/project-system" - _targetSolution: "ProjectSystem.sln" - _branchName: "main" - _sha: "e660d54d6b3198751bd0502fe270e1657f32a913" - _useParentSdk: 1 - - Name: msbuild - _repo: "https://github.com/dotnet/msbuild" - _repoName: "dotnet/msbuild" - _targetSolution: "MSBuild.sln" - _branchName: "main" - _sha: "f4fa6bde775a3f7cbb2bb90a349ee5fc759114f3" - _useParentSdk: 0 - - Name: aspnetcore - _repo: "https://github.com/dotnet/aspnetcore" - _repoName: "dotnet/aspnetcore" - _targetSolution: "AspNetCore.sln" - _branchName: "main" - _sha: "d765d7ba4871a8c2cb38d4134553d3be9a7370d7" - _useParentSdk: 0 - - Name: efcore - _repo: "https://github.com/dotnet/efcore" - _repoName: "dotnet/efcore" - _targetSolution: "All.sln" - _branchName: "main" - _sha: "1b2ff365399ab6736a9ea4c98ab1b60acda5d917" - _useParentSdk: 0 - - Name: razor_tooling - _repo: "https://github.com/dotnet/razor" - _repoName: "dotnet/razor" - _targetSolution: "Razor.sln" - _branchName: "main" - _sha: "ecb4b595e3322a18c240f50a763868540f51eaaa" - _useParentSdk: 0 +- name: oneESCompat + type: object + default: + publishTaskName: PublishBuildArtifacts@1 + +- name: TestArguments + type: object + default: + - Name: Roslyn + _repo: "https://github.com/dotnet/roslyn" + _repoName: "dotnet/roslyn" + _targetSolution: "Compilers.slnf" + _branchName: "main" + _sha: "a3bb37003aeccad012a6e7dd220977599e8b8e65" + _useParentSdk: 0 + - Name: sdk + _repo: "https://github.com/dotnet/sdk" + _repoName: "dotnet/sdk" + _targetSolution: "sdk.sln" + _branchName: "main" + _sha: "be25db95c376bffd508a023399ddd34392fe6458" + _useParentSdk: 0 + - Name: project_system + _repo: "https://github.com/dotnet/project-system" + _repoName: "dotnet/project-system" + _targetSolution: "ProjectSystem.sln" + _branchName: "main" + _sha: "e660d54d6b3198751bd0502fe270e1657f32a913" + _useParentSdk: 1 + - Name: msbuild + _repo: "https://github.com/dotnet/msbuild" + _repoName: "dotnet/msbuild" + _targetSolution: "MSBuild.sln" + _branchName: "main" + _sha: "f4fa6bde775a3f7cbb2bb90a349ee5fc759114f3" + _useParentSdk: 0 + - Name: aspnetcore + _repo: "https://github.com/dotnet/aspnetcore" + _repoName: "dotnet/aspnetcore" + _targetSolution: "AspNetCore.sln" + _branchName: "main" + _sha: "d765d7ba4871a8c2cb38d4134553d3be9a7370d7" + _useParentSdk: 0 + - Name: efcore + _repo: "https://github.com/dotnet/efcore" + _repoName: "dotnet/efcore" + _targetSolution: "All.sln" + _branchName: "main" + _sha: "1b2ff365399ab6736a9ea4c98ab1b60acda5d917" + _useParentSdk: 0 + - Name: razor_tooling + _repo: "https://github.com/dotnet/razor" + _repoName: "dotnet/razor" + _targetSolution: "Razor.sln" + _branchName: "main" + _sha: "ecb4b595e3322a18c240f50a763868540f51eaaa" + _useParentSdk: 0 jobs: - job: Formatting_Check + displayName: Run Formatting Check pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) @@ -66,22 +70,23 @@ jobs: os: windows timeoutInMinutes: 60 steps: - - script: .\restore.cmd - displayName: Restore dependencies - - script: | - .\artifacts\sdk-build-env.bat - dotnet run --project .\src\BuiltInTools\dotnet-format\dotnet-format.csproj -c Release -- @eng\dotnet-format\validate.rsp - displayName: Run dotnet-format - - task: ${{ parameters.PublishTaskName }} - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'dotnet-format formatting Check' - continueOnError: true - condition: not(succeeded()) + - script: .\restore.cmd + displayName: Restore dependencies + - script: | + .\artifacts\sdk-build-env.bat + dotnet run --project .\src\BuiltInTools\dotnet-format\dotnet-format.csproj -c Release -- @eng\dotnet-format\validate.rsp + displayName: Run dotnet-format + - task: ${{ parameters.oneESCompat.publishTaskName }} + displayName: Publish Logs + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: dotnet-format formatting Check + continueOnError: true + condition: not(succeeded()) - ${{ each testArgs in parameters.TestArguments }}: - job: dotnet_format_integration_tests_${{ testArgs.Name }} + displayName: 'Dotnet-Format Integration Tests: ${{ testArgs.Name }}' pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) @@ -92,11 +97,11 @@ jobs: os: windows timeoutInMinutes: 60 steps: - - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'prepare' - displayName: Prepare ${{ testArgs._repoName }} for formatting + - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'prepare' + displayName: Prepare ${{ testArgs._repoName }} for formatting - - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'format-workspace' - displayName: Run dotnet-format on ${{ testArgs._repoName }} ${{ testArgs._targetSolution }} + - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'format-workspace' + displayName: Run dotnet-format on ${{ testArgs._repoName }} ${{ testArgs._targetSolution }} - - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'format-folder' - displayName: Run dotnet-format on ${{ testArgs._repoName }} repo folder + - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'format-folder' + displayName: Run dotnet-format on ${{ testArgs._repoName }} repo folder diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 55b26a9a98b5..17e2477c2cab 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,6 +1,6 @@ parameters: variables: {} - oneEsPtCompat: + oneESCompat: jobTemplatePath: /eng/common/templates/job/job.yml publishTaskName: PublishBuildArtifacts@1 container: '' @@ -22,7 +22,7 @@ parameters: officialBuildProperties: '' jobs: -- template: ${{ parameters.oneEsPtCompat.jobTemplatePath }} +- template: ${{ parameters.oneESCompat.jobTemplatePath }} parameters: ${{ if eq(parameters.container, '') }}: name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} @@ -107,6 +107,7 @@ jobs: ############### TESTING ############### - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: + # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload - ${{ if eq(parameters.pool.os, 'windows') }}: @@ -145,58 +146,6 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) RunAoTTests: ${{ parameters.runAoTTests }} - # ############### NON-WINDOWS ############### - # - ${{ else }}: - # ############### BUILDING ############### - # - ${{ if ne(variables['System.TeamProject'], 'public') }}: - # - script: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $Token - # displayName: Setup Private Feeds Credentials - # env: - # Token: $(dn-bot-dnceng-artifact-feeds-rw) - - # - powershell: eng/common/build.ps1 - # -restore -build -sign -pack -ci -nativeToolsOnMachine - # -configuration $(buildConfiguration) - # $(publishArgument) - # /p:Architecture=$(buildArchitecture) - # $(runtimeSourceProperties) - # $(testInBuildProperties) - # $(officialBuildProperties) - # displayName: Build - # env: - # # Required by Arcade for passing to Helix - # SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # # Only set when ran internally - # HelixAccessToken: $(HelixApiAccessToken) - # BuildConfig: $(buildConfiguration) - # TestFullMSBuild: $(testFullMSBuild) - - # ############### TESTING ############### - # # TODO: Change _CustomHelixTargetQueue name - # - powershell: eng/common/build.ps1 - # -restore -test -ci -nativeToolsOnMachine -prepareMachine - # -configuration $(buildConfiguration) - # -projects $(testProjects) - # /p:Architecture=$(buildArchitecture) - # $(runtimeSourceProperties) - # /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - # /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/$(testName).binlog - # displayName: Run $(testName) - # condition: and(succeeded(), eq(variables.runTests, true)) - # env: - # # Required by Arcade for passing to Helix - # SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # # Only set when ran internally - # HelixAccessToken: $(HelixApiAccessToken) - # RunAoTTests: $(runAoTTests) - - - - - - - - ############### POST ############### - task: CopyFiles@2 displayName: Copy Logs @@ -217,7 +166,7 @@ jobs: continueOnError: true condition: always() - - task: ${{ parameters.oneEsPtCompat.publishTaskName }} + - task: ${{ parameters.oneESCompat.publishTaskName }} displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 6c775c295877..2831a99c6d73 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -1,198 +1,31 @@ -# parameters: - # jobTemplatePath: /eng/common/templates/job/job.yml - # publishTaskName: PublishBuildArtifacts@1 - # buildCategory: Build - # buildArchitecture: x64 - # # Agent OS identifier and used as job name - # agentOs: '' - # # Agent pool - # pool: {} - # # Additional variables - # variables: {} - # # Build strategy - matrix - # strategy: '' - - # testFullMSBuild: false - # publishArgs: '' - # signType: test - # buildConfiguration: Debug - # testArgs: -test - # additionalArgs: '' - - - # linuxContainerInfo: - # ${{ if eq(parameters.runTests, false) }}: - - - # _publishArgument: -publish - # _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - # _windowsJobParameters: - # - ${{ if eq(parameters.runTests, false) }}: - # - categoryName: Official - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: x86 - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - categoryName: Official - # buildArchitecture: arm64 - # publishArgument: ${{ parameters._publishArgument }} - # officialBuildProperties: ${{ parameters._officialBuildProperties }} - # variables: - # _SignType: real - # - ${{ else }}: - # - categoryName: TestBuild - # - categoryName: FullFramework - # testFullMSBuild: true - # - categoryName: TestAsTools - # runTestsAsTool: true - # # This uses the build step for testing, so the extra test step is not necessary. - # runTests: false - # - categoryName: TemplateEngine - # testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - # $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - # - categoryName: AoT - # runAoTTests: true - parameters: jobParameterSets: [] + publishRetryConfig: false + locBranch: '' jobs: +- ${{ if eq(parameters.publishRetryConfig, true) }}: + # Publishes the build-configuration.json to the artifacts which Arcade uses to allow the jobs in the pipeline run to retry automatically. + # https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/BuildRetryOnboard.md + - job: PublishRetryConfig + displayName: Publish Pipeline Retry Config + pool: ${{ parameters.pool }} + steps: + - task: ${{ parameters.oneESCompat.publishTaskName }} + displayName: Publish build-configuration.json + inputs: + targetPath: $(Build.SourcesDirectory)/eng/BuildConfiguration + artifactName: BuildConfiguration +- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: + - template: /eng/common/templates-official/job/onelocbuild.yml + parameters: + CreatePr: true + LclSource: lclFilesfromPackage + LclPackageId: LCL-JUNO-PROD-DOTNETSDK + MirrorBranch: ${{ parameters.locBranch }} + MirrorRepo: sdk - ${{ each jobParameters in parameters.jobParameterSets }}: - - template: /eng/pipelines/templates/jobs/sdk-build.yml@self + - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: ${{ insert }}: ${{ parameters }} - ${{ insert }}: ${{ jobParameters }} - - - -############### WINDOWS ############### -# - ${{ each jobParameters in parameters._windowsJobParameters }}: -# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self -# parameters: -# ${{ insert }}: ${{ jobParameters }} -# oneEsPtCompat: ${{ parameters.oneEsPtCompat }} -# runTests: ${{ parameters.runTests }} - - - - -# ############### WINDOWS ############### -# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self -# parameters: -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 -# os: windows -# helixTargetQueue: windows.amd64.vs2022.pre -# oneEsPtCompat: -# jobTemplatePath: /eng/common/templates-official/job/job.yml -# publishTaskName: 1ES.PublishBuildArtifacts@1 -# variables: -# runTests: ${{ parameters.runTests }} -# # Setting this here once so it can be reused in the matrix only. -# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) -# strategy: -# matrix: -# ${{ if eq(parameters.runTests, false) }}: -# Official: -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# Official_x86: -# buildArchitecture: x86 -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# Official_arm64: -# buildArchitecture: arm64 -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# ${{ else }}: -# Test: {} -# FullFramework: -# testFullMSBuild: true -# TestAsTools: -# testInBuildProperties: /p:RunTestsAsTool=true -# # This uses the build step for testing, so the extra test step is not necessary. -# runTests: false -# TemplateEngine: -# testName: TemplateEngineTests -# testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; -# $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj -# AoT: -# runAoTTests: true - -# ############### LINUX ############### -# # TODO: Need to foreach this entire section for each container. But only do that once this logic is in the job-matrix file. -# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self -# parameters: -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-ubuntu-2204 -# os: linux -# helixTargetQueue: ubuntu.2204.amd64 -# container: ubuntu2204 -# oneEsPtCompat: -# jobTemplatePath: /eng/common/templates-official/job/job.yml -# publishTaskName: 1ES.PublishBuildArtifacts@1 -# variables: -# runTests: ${{ parameters.runTests }} -# # Setting this here once so it can be reused in the matrix only. -# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) -# strategy: -# matrix: -# ${{ if eq(parameters.runTests, false) }}: -# Official: -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# Official_x86: -# buildArchitecture: x86 -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# Official_arm64: -# buildArchitecture: arm64 -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# ${{ else }}: -# Test: {} -# Test_arm64: -# testFullMSBuild: true - -# ############### MACOS ############### -# - template: /eng/pipelines/templates/jobs/sdk-build.yml@self -# parameters: -# pool: -# name: Azure Pipelines -# image: macOS-latest -# os: macOS -# helixTargetQueue: OSX.13.Amd64 -# oneEsPtCompat: -# jobTemplatePath: /eng/common/templates-official/job/job.yml -# publishTaskName: 1ES.PublishBuildArtifacts@1 -# variables: -# runTests: ${{ parameters.runTests }} -# # Setting this here once so it can be reused in the matrix only. -# _officialBuildProperties: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) -# strategy: -# matrix: -# ${{ if eq(parameters.runTests, false) }}: -# Official: -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) -# Official_arm64: -# runtimeIdentifierProperties: /p:Rid=osx-arm64 -# buildArchitecture: arm64 -# publishArgument: -publish -# _SignType: real -# officialBuildProperties: $(_officialBuildProperties) \ No newline at end of file + ${{ insert }}: ${{ jobParameters }} \ No newline at end of file From 78fd4d1926c064293358c94da3759b0dcbf530aa Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:27:41 -0700 Subject: [PATCH 51/90] Comment out source build and dotnet format again. --- .vsts-ci.yml | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e1796208fbc1..fd61708e006e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -345,35 +345,19 @@ extends: - categoryName: AoT runAoTTests: true - ############### SOURCE BUILD ############### - - template: /eng/common/templates-official/job/source-build.yml@self - parameters: - platform: - name: Managed - container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + # - template: /eng/common/templates-official/job/source-build.yml@self + # parameters: + # platform: + # name: Managed + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 ############### DOTNET-FORMAT ############### - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/dotnet-format/dotnet-format-integration.yml@self - parameters: - oneESCompat: - publishTaskName: 1ES.PublishBuildArtifacts@1 - - - # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - # - template: /eng/common/templates-official/job/publish-build-assets.yml@self - # parameters: - # publishUsingPipelines: true - # publishAssetsImmediately: true - # dependsOn: - # - Windows_NT - # - Source_Build_Managed - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-windows-2022 - # os: windows - + # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # - template: /eng/dotnet-format/dotnet-format-integration.yml@self + # parameters: + # oneESCompat: + # publishTaskName: 1ES.PublishBuildArtifacts@1 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - stage: publish From 3aa292638eb7ae58fa2e06b0a03863f1164c6339 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:30:15 -0700 Subject: [PATCH 52/90] Comment out onelocbuild. --- eng/pipelines/templates/jobs/sdk-job-matrix.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 2831a99c6d73..4d14cb2b501d 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -16,14 +16,15 @@ jobs: inputs: targetPath: $(Build.SourcesDirectory)/eng/BuildConfiguration artifactName: BuildConfiguration -- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: - - template: /eng/common/templates-official/job/onelocbuild.yml - parameters: - CreatePr: true - LclSource: lclFilesfromPackage - LclPackageId: LCL-JUNO-PROD-DOTNETSDK - MirrorBranch: ${{ parameters.locBranch }} - MirrorRepo: sdk +# TODO: This needs oneESCompat value. +# - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: +# - template: /eng/common/templates-official/job/onelocbuild.yml +# parameters: +# CreatePr: true +# LclSource: lclFilesfromPackage +# LclPackageId: LCL-JUNO-PROD-DOTNETSDK +# MirrorBranch: ${{ parameters.locBranch }} +# MirrorRepo: sdk - ${{ each jobParameters in parameters.jobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: From 9515ff20b9f11dbe7b243a9d0fa7dd86b5b04231 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:33:39 -0700 Subject: [PATCH 53/90] Commenting out post-build. --- .vsts-ci.yml | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index fd61708e006e..443d9ccc6e27 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -373,25 +373,24 @@ extends: image: 1es-windows-2022 os: windows - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates-official/post-build/post-build.yml@self - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "dotnet-sdk" - -TsaCodebaseName "dotnet-sdk" - -TsaPublish $True' + # - template: /eng/common/templates-official/post-build/post-build.yml@self + # parameters: + # publishingInfraVersion: 3 + # enableSymbolValidation: false + # enableSigningValidation: false + # enableNugetValidation: false + # enableSourceLinkValidation: false + # publishInstallersAndChecksums: true + # publishAssetsImmediately: true + # SDLValidationParameters: + # enable: false + # params: ' -SourceToolsList @("policheck","credscan") + # -TsaInstanceURL $(_TsaInstanceURL) + # -TsaProjectName $(_TsaProjectName) + # -TsaNotificationEmail $(_TsaNotificationEmail) + # -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + # -TsaBugAreaPath $(_TsaBugAreaPath) + # -TsaIterationPath $(_TsaIterationPath) + # -TsaRepositoryName "dotnet-sdk" + # -TsaCodebaseName "dotnet-sdk" + # -TsaPublish $True' From c50ed05d1f55acfa8c189675a2511d89275b4824 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:34:39 -0700 Subject: [PATCH 54/90] Commenting out publish-build-assets. --- .vsts-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 443d9ccc6e27..651a519d7cac 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -360,18 +360,18 @@ extends: # publishTaskName: 1ES.PublishBuildArtifacts@1 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - stage: publish - displayName: Publish - dependsOn: build - jobs: - - template: /eng/common/templates-official/job/publish-build-assets.yml@self - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - pool: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows + # - stage: publish + # displayName: Publish + # dependsOn: build + # jobs: + # - template: /eng/common/templates-official/job/publish-build-assets.yml@self + # parameters: + # publishUsingPipelines: true + # publishAssetsImmediately: true + # pool: + # name: $(DncEngInternalBuildPool) + # image: 1es-windows-2022 + # os: windows # - template: /eng/common/templates-official/post-build/post-build.yml@self # parameters: From cdc29b449d38c41e9ab1ae09c446472e5c3f133f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:35:25 -0700 Subject: [PATCH 55/90] Comment if statement. --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 651a519d7cac..d1d15b5bddcf 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -359,7 +359,7 @@ extends: # oneESCompat: # publishTaskName: 1ES.PublishBuildArtifacts@1 - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: # - stage: publish # displayName: Publish # dependsOn: build From 24dd70fdf457f79ebf7d0ad8d2eec2ece5bd7aac Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:51:55 -0700 Subject: [PATCH 56/90] Trying to use templateFolderName for partial path. Trying to fix log artifact name. Enabled commented out blocks. Changed Publish retry config to use the PublishBuildArtifact instead of PublishPipelineArtifact. --- .vsts-ci.yml | 97 ++++++++++--------- eng/pipelines/templates/jobs/sdk-build.yml | 15 ++- .../templates/jobs/sdk-job-matrix.yml | 20 ++-- 3 files changed, 70 insertions(+), 62 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index d1d15b5bddcf..120ad355fdc0 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -107,7 +107,8 @@ extends: os: windows helixTargetQueue: windows.amd64.vs2022.pre oneESCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml + # jobTemplatePath: /eng/common/templates-official/job/job.yml + templateFolderName: templates-official publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) publishRetryConfig: true @@ -189,7 +190,8 @@ extends: os: linux helixTargetQueue: ubuntu.2204.amd64 oneESCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml + # jobTemplatePath: /eng/common/templates-official/job/job.yml + templateFolderName: templates-official publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: @@ -319,7 +321,8 @@ extends: os: macOS helixTargetQueue: OSX.13.Amd64 oneESCompat: - jobTemplatePath: /eng/common/templates-official/job/job.yml + # jobTemplatePath: /eng/common/templates-official/job/job.yml + templateFolderName: templates-official publishTaskName: 1ES.PublishBuildArtifacts@1 runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: @@ -346,51 +349,51 @@ extends: runAoTTests: true ############### SOURCE BUILD ############### - # - template: /eng/common/templates-official/job/source-build.yml@self - # parameters: - # platform: - # name: Managed - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + - template: /eng/common/templates-official/job/source-build.yml@self + parameters: + platform: + name: Managed + container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 ############### DOTNET-FORMAT ############### - # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - template: /eng/dotnet-format/dotnet-format-integration.yml@self - # parameters: - # oneESCompat: - # publishTaskName: 1ES.PublishBuildArtifacts@1 + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/dotnet-format/dotnet-format-integration.yml@self + parameters: + oneESCompat: + publishTaskName: 1ES.PublishBuildArtifacts@1 - # - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - # - stage: publish - # displayName: Publish - # dependsOn: build - # jobs: - # - template: /eng/common/templates-official/job/publish-build-assets.yml@self - # parameters: - # publishUsingPipelines: true - # publishAssetsImmediately: true - # pool: - # name: $(DncEngInternalBuildPool) - # image: 1es-windows-2022 - # os: windows + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - stage: publish + displayName: Publish + dependsOn: build + jobs: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows - # - template: /eng/common/templates-official/post-build/post-build.yml@self - # parameters: - # publishingInfraVersion: 3 - # enableSymbolValidation: false - # enableSigningValidation: false - # enableNugetValidation: false - # enableSourceLinkValidation: false - # publishInstallersAndChecksums: true - # publishAssetsImmediately: true - # SDLValidationParameters: - # enable: false - # params: ' -SourceToolsList @("policheck","credscan") - # -TsaInstanceURL $(_TsaInstanceURL) - # -TsaProjectName $(_TsaProjectName) - # -TsaNotificationEmail $(_TsaNotificationEmail) - # -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - # -TsaBugAreaPath $(_TsaBugAreaPath) - # -TsaIterationPath $(_TsaIterationPath) - # -TsaRepositoryName "dotnet-sdk" - # -TsaCodebaseName "dotnet-sdk" - # -TsaPublish $True' + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + enableSourceLinkValidation: false + publishInstallersAndChecksums: true + publishAssetsImmediately: true + SDLValidationParameters: + enable: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "dotnet-sdk" + -TsaCodebaseName "dotnet-sdk" + -TsaPublish $True' diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 17e2477c2cab..1d102c78d79b 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,7 +1,8 @@ parameters: variables: {} oneESCompat: - jobTemplatePath: /eng/common/templates/job/job.yml + # jobTemplatePath: /eng/common/templates/job/job.yml + templateFolderName: templates publishTaskName: PublishBuildArtifacts@1 container: '' categoryName: Build @@ -22,14 +23,14 @@ parameters: officialBuildProperties: '' jobs: -- template: ${{ parameters.oneESCompat.jobTemplatePath }} +- template: /eng/common/${{ parameters.templateFolderName }}/job/job.yml parameters: ${{ if eq(parameters.container, '') }}: name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - displayName: "${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }})" + displayName: '${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }})' ${{ else }}: name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} - displayName: "${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }}) [${{ parameters.container }}]" + displayName: '${{ parameters.categoryName }}: ${{ parameters.pool.os }} (${{ parameters.buildArchitecture }}) [${{ parameters.container }}]' pool: ${{ parameters.pool }} container: ${{ parameters.container }} strategy: ${{ parameters.strategy }} @@ -171,7 +172,11 @@ jobs: inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) # TODO: Figure out appropriate artifact name - ArtifactName: ${{ parameters.name }} + # ArtifactName: ${{ parameters.name }} + ${{ if eq(parameters.container, '') }}: + ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + ${{ else }}: + ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} publishLocation: Container continueOnError: true condition: always() diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 4d14cb2b501d..e2e724396576 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -14,17 +14,17 @@ jobs: - task: ${{ parameters.oneESCompat.publishTaskName }} displayName: Publish build-configuration.json inputs: - targetPath: $(Build.SourcesDirectory)/eng/BuildConfiguration - artifactName: BuildConfiguration + PathtoPublish: $(Build.SourcesDirectory)/eng/BuildConfiguration + ArtifactName: BuildConfiguration # TODO: This needs oneESCompat value. -# - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: -# - template: /eng/common/templates-official/job/onelocbuild.yml -# parameters: -# CreatePr: true -# LclSource: lclFilesfromPackage -# LclPackageId: LCL-JUNO-PROD-DOTNETSDK -# MirrorBranch: ${{ parameters.locBranch }} -# MirrorRepo: sdk +- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: + - template: /eng/common/${{ parameters.templateFolderName }}/job/onelocbuild.yml + parameters: + CreatePr: true + LclSource: lclFilesfromPackage + LclPackageId: LCL-JUNO-PROD-DOTNETSDK + MirrorBranch: ${{ parameters.locBranch }} + MirrorRepo: sdk - ${{ each jobParameters in parameters.jobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: From 30308e103493dd1d3f0e769c5ebbff58aa4f842e Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 13:54:25 -0700 Subject: [PATCH 57/90] Specified the template folder name improperly. --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- eng/pipelines/templates/jobs/sdk-job-matrix.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 1d102c78d79b..7d9229502bdf 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -23,7 +23,7 @@ parameters: officialBuildProperties: '' jobs: -- template: /eng/common/${{ parameters.templateFolderName }}/job/job.yml +- template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/job/job.yml parameters: ${{ if eq(parameters.container, '') }}: name: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index e2e724396576..f31e30322ccb 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -16,9 +16,8 @@ jobs: inputs: PathtoPublish: $(Build.SourcesDirectory)/eng/BuildConfiguration ArtifactName: BuildConfiguration -# TODO: This needs oneESCompat value. - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: - - template: /eng/common/${{ parameters.templateFolderName }}/job/onelocbuild.yml + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/job/onelocbuild.yml parameters: CreatePr: true LclSource: lclFilesfromPackage From 87c6e5c4dbe512c0904d334bacf16f3b69f3fd77 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 15:11:53 -0700 Subject: [PATCH 58/90] Changed to using publishTaskPrefix. Set EnableDefaultArtifacts to false for non-Windows builds. --- .vsts-ci.yml | 19 +++++++------------ .../dotnet-format-integration.yml | 4 ++-- eng/pipelines/templates/jobs/sdk-build.yml | 15 ++++++++------- .../templates/jobs/sdk-job-matrix.yml | 5 ++++- .../templates/variables/sdk-defaults.yml | 15 --------------- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 120ad355fdc0..81e28e4080e3 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -21,11 +21,6 @@ parameters: displayName: Run Tests type: boolean default: false - # 💢 "A template expression is not allowed in this context" - # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # default: true - # ${{ else }}: - # default: false variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml @@ -95,6 +90,7 @@ extends: alertWarningLevel: Critical stages: + ############### BUILD STAGE ############### - stage: build displayName: Build jobs: @@ -107,9 +103,8 @@ extends: os: windows helixTargetQueue: windows.amd64.vs2022.pre oneESCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml templateFolderName: templates-official - publishTaskName: 1ES.PublishBuildArtifacts@1 + publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) publishRetryConfig: true locBranch: release/8.0.4xx @@ -190,9 +185,8 @@ extends: os: linux helixTargetQueue: ubuntu.2204.amd64 oneESCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml templateFolderName: templates-official - publishTaskName: 1ES.PublishBuildArtifacts@1 + publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -321,9 +315,8 @@ extends: os: macOS helixTargetQueue: OSX.13.Amd64 oneESCompat: - # jobTemplatePath: /eng/common/templates-official/job/job.yml templateFolderName: templates-official - publishTaskName: 1ES.PublishBuildArtifacts@1 + publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) jobParameterSets: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -360,8 +353,9 @@ extends: - template: /eng/dotnet-format/dotnet-format-integration.yml@self parameters: oneESCompat: - publishTaskName: 1ES.PublishBuildArtifacts@1 + publishTaskPrefix: 1ES. + ############### PUBLISH STAGE ############### - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - stage: publish displayName: Publish @@ -376,6 +370,7 @@ extends: image: 1es-windows-2022 os: windows + ############### POST-BUILD STAGE ############### - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: publishingInfraVersion: 3 diff --git a/eng/dotnet-format/dotnet-format-integration.yml b/eng/dotnet-format/dotnet-format-integration.yml index bff84d7c707e..602b2d7d2b01 100644 --- a/eng/dotnet-format/dotnet-format-integration.yml +++ b/eng/dotnet-format/dotnet-format-integration.yml @@ -2,7 +2,7 @@ parameters: - name: oneESCompat type: object default: - publishTaskName: PublishBuildArtifacts@1 + publishTaskPrefix: '' - name: TestArguments type: object @@ -76,7 +76,7 @@ jobs: .\artifacts\sdk-build-env.bat dotnet run --project .\src\BuiltInTools\dotnet-format\dotnet-format.csproj -c Release -- @eng\dotnet-format\validate.rsp displayName: Run dotnet-format - - task: ${{ parameters.oneESCompat.publishTaskName }} + - task: ${{ parameters.oneESCompat.publishTaskPrefix }}PublishBuildArtifacts@1 displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 7d9229502bdf..9354f7f924a0 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,9 +1,8 @@ parameters: variables: {} oneESCompat: - # jobTemplatePath: /eng/common/templates/job/job.yml templateFolderName: templates - publishTaskName: PublishBuildArtifacts@1 + publishTaskPrefix: '' container: '' categoryName: Build runTests: true @@ -89,6 +88,7 @@ jobs: -configuration $(buildConfiguration) \ ${{ parameters.publishArgument }} \ ${{ parameters.signArgument }} \ + /p:EnableDefaultArtifacts=false \ /p:Architecture=${{ parameters.buildArchitecture }} \ /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ /p:PgoInstrument=${{ parameters.pgoInstrument }} \ @@ -167,16 +167,17 @@ jobs: continueOnError: true condition: always() - - task: ${{ parameters.oneESCompat.publishTaskName }} + - task: ${{ parameters.oneESCompat.publishTaskPrefix }}PublishBuildArtifacts@1 displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) # TODO: Figure out appropriate artifact name + ArtifactName: $(System.PhaseName) # ArtifactName: ${{ parameters.name }} - ${{ if eq(parameters.container, '') }}: - ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - ${{ else }}: - ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} + # ${{ if eq(parameters.container, '') }}: + # ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} + # ${{ else }}: + # ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} publishLocation: Container continueOnError: true condition: always() diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index f31e30322ccb..5e6756745f77 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -1,5 +1,8 @@ parameters: jobParameterSets: [] + oneESCompat: + templateFolderName: templates + publishTaskPrefix: '' publishRetryConfig: false locBranch: '' @@ -11,7 +14,7 @@ jobs: displayName: Publish Pipeline Retry Config pool: ${{ parameters.pool }} steps: - - task: ${{ parameters.oneESCompat.publishTaskName }} + - task: ${{ parameters.oneESCompat.publishTaskPrefix }}PublishBuildArtifacts@1 displayName: Publish build-configuration.json inputs: PathtoPublish: $(Build.SourcesDirectory)/eng/BuildConfiguration diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 5936171d7c9f..56c2400a0497 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,21 +1,6 @@ variables: ############### BUILD ############### buildConfiguration: Release - # buildArchitecture: x64 - # runTests: true - # testFullMSBuild: false - # runAoTTests: false - - # TODO: Instead, just have this always on the build and set the value, same with RunTestsAsTool. - # testInBuildProperties: /p:Test=false - # testName: HelixTests - # testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj - - # runtimeSourceProperties: '' - # runtimeIdentifierProperties: '' - - # publishArgument: '' - # officialBuildProperties: '' ############### ARCADE ############### # "Post-build signing" means that this build will NOT sign itself. From cf7da3ef16ddd390dfe4fdc8d931c6da92dc4a9a Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 13 May 2024 15:34:50 -0700 Subject: [PATCH 59/90] Added EnableToolsetPublish to only publish the toolset nupkg once. --- .vsts-ci.yml | 5 +++-- eng/Publishing.props | 2 +- eng/pipelines/templates/variables/sdk-defaults.yml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 81e28e4080e3..e60dd142e00a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -88,6 +88,7 @@ extends: # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ alertWarningLevel: Critical + ignoreDirectories: artifacts, .packages stages: ############### BUILD STAGE ############### @@ -112,7 +113,7 @@ extends: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: ### OFFICIAL ### - categoryName: Official - publishArgument: $(_publishArgument) /p:PublishInternalAsset=true + publishArgument: $(_publishArgument) /p:EnableToolsetPublish=true signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false @@ -137,7 +138,7 @@ extends: ### PGO ### - categoryName: PGO pgoInstrument: true - publishArgument: $(_publishArgument) /p:PublishInternalAsset=true + publishArgument: $(_publishArgument) signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false diff --git a/eng/Publishing.props b/eng/Publishing.props index e5a7e8b47f7e..8697cfe46ca3 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -95,7 +95,7 @@ - + Date: Tue, 14 May 2024 10:51:47 -0700 Subject: [PATCH 60/90] Enabled publishing for installer assets. Testing component governance ignore. --- .vsts-ci.yml | 2 +- eng/Publishing.props | 12 +++++------- eng/pipelines/templates/jobs/sdk-build.yml | 8 ++------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e60dd142e00a..1197c2f13208 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -87,7 +87,7 @@ extends: # Setting to Critical as aspnet templates currently cause a High alert (default alert level is High). # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - alertWarningLevel: Critical + # alertWarningLevel: Critical ignoreDirectories: artifacts, .packages stages: diff --git a/eng/Publishing.props b/eng/Publishing.props index 8697cfe46ca3..9b70d0876e26 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -184,17 +184,15 @@ - - - - + + + + - - + diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 9354f7f924a0..d1be98f73783 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -171,13 +171,9 @@ jobs: displayName: Publish Logs inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) - # TODO: Figure out appropriate artifact name + # This is the job name, but because of "legacy reasons", both Agent.JobName and System.JobName are not the actual job name. + # See: https://developercommunity.visualstudio.com/t/systemjobname-seems-to-be-incorrectly-assigned-and/1209736#TPIN-N1211828 ArtifactName: $(System.PhaseName) - # ArtifactName: ${{ parameters.name }} - # ${{ if eq(parameters.container, '') }}: - # ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }} - # ${{ else }}: - # ArtifactName: ${{ parameters.categoryName }}_${{ parameters.pool.os }}_${{ parameters.buildArchitecture }}_${{ parameters.container }} publishLocation: Container continueOnError: true condition: always() From 9798e19e9055dede7ea940d5801c0b5d6df5cb9d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 14 May 2024 14:09:52 -0700 Subject: [PATCH 61/90] Use centosStream8 instead. Fix aot command path. Trying to fix test project specification. --- .vsts-ci.yml | 16 +++++++--------- eng/pipelines/templates/jobs/sdk-build.yml | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1197c2f13208..ed27327a6dca 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -66,8 +66,9 @@ extends: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode cblMariner20Fpm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - centosStream9: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + # WORKAROUND: The centosStream9 assets are not published. So, we need to revert to 8 for the time being. + centosStream8: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 debian11Amd64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 fedora39: @@ -172,8 +173,7 @@ extends: # This uses the build step for testing, so the extra test step is not necessary. runTests: false - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true @@ -292,7 +292,7 @@ extends: container: fedora39 osProperties: $(_linuxOsProperties) - categoryName: TestBuild - container: centosStream9 + container: centosStream8 osProperties: /p:OSName=linux - categoryName: TestBuild container: debian11Amd64 @@ -304,8 +304,7 @@ extends: osProperties: /p:HostOSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -337,8 +336,7 @@ extends: ### TESTS ### - categoryName: TestBuild - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj; - $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index d1be98f73783..c30e16978ad1 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -109,14 +109,14 @@ jobs: - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update + - script: $(Build.SourcesDirectory)/artifacts/bin/redist-installer/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload - ${{ if eq(parameters.pool.os, 'windows') }}: # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) - -projects ${{ parameters.testProjects }} + -projects "${{ parameters.testProjects }}" /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} @@ -132,7 +132,7 @@ jobs: - script: eng/common/build.sh -restore -test -ci -prepareMachine -configuration $(buildConfiguration) - -projects ${{ parameters.testProjects }} + -projects "${{ parameters.testProjects }}" /p:Architecture=${{ parameters.buildArchitecture }} /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} From a9343314a0813595b4f68c3f37bf53d08f1453ed Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 14 May 2024 15:32:13 -0700 Subject: [PATCH 62/90] Added missing PgoTerm logic for Directory.Build.props. Fix AoT workload line with buildConfiguration. Trying to fix TemplateEngine test project specification. --- .vsts-ci.yml | 3 --- Directory.Build.props | 5 +++++ eng/pipelines/templates/jobs/sdk-build.yml | 6 +++--- eng/pipelines/templates/variables/sdk-defaults.yml | 5 +---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index ed27327a6dca..e3c5534ee60a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -85,10 +85,7 @@ extends: image: 1es-windows-2022 os: windows componentgovernance: - # Setting to Critical as aspnet templates currently cause a High alert (default alert level is High). - # See: https://dev.azure.com/dnceng/internal/_componentGovernance/dotnet-sdk/alert/9439021?typeId=6422467 # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - # alertWarningLevel: Critical ignoreDirectories: artifacts, .packages stages: diff --git a/Directory.Build.props b/Directory.Build.props index 232846cb33f2..2c0593ea07fd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -56,6 +56,11 @@ true + + true + -pgo + + false false diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index c30e16978ad1..40b98f54ffc9 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -109,14 +109,14 @@ jobs: - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - - script: $(Build.SourcesDirectory)/artifacts/bin/redist-installer/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update + - script: $(Build.SourcesDirectory)/artifacts/bin/redist-installer/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload - ${{ if eq(parameters.pool.os, 'windows') }}: # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) - -projects "${{ parameters.testProjects }}" + -projects ${{ parameters.testProjects }} /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} @@ -132,7 +132,7 @@ jobs: - script: eng/common/build.sh -restore -test -ci -prepareMachine -configuration $(buildConfiguration) - -projects "${{ parameters.testProjects }}" + -projects ${{ parameters.testProjects }} /p:Architecture=${{ parameters.buildArchitecture }} /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 7b20e654173c..7f7a0b5d2d61 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -9,7 +9,4 @@ variables: _SignType: test _TeamName: DotNetCore # This ends up being set by the DotNet-HelixApi-Access variable group when running internally. - HelixApiAccessToken: '' - - ############### OTHER ############### - # Codeql.ExcludePathPatterns: artifacts, .packages \ No newline at end of file + HelixApiAccessToken: '' \ No newline at end of file From a4006bb7db0d3615f40982356c803b390373ee9b Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 14 May 2024 15:57:53 -0700 Subject: [PATCH 63/90] Trying to escape the semicolon. --- .vsts-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e3c5534ee60a..b85e2240e073 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -170,7 +170,8 @@ extends: # This uses the build step for testing, so the extra test step is not necessary. runTests: false - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use backtick (`) to escape the semicolon for the PowerShell task. + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj`;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true @@ -301,7 +302,8 @@ extends: osProperties: /p:HostOSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use backslash (\) to escape the semicolon for the script task. + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj\;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -333,7 +335,8 @@ extends: ### TESTS ### - categoryName: TestBuild - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use backslash (\) to escape the semicolon for the script task. + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj\;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true From 950105f533b5f9d77a605eb1150e63cf9f759589 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 15 May 2024 10:17:53 -0700 Subject: [PATCH 64/90] Changed _CustomHelixTargetQueue to CustomHelixTargetQueue. Made AoT wasm path use non-installer dotnet version. Using encoding for semicolon so TemplateEngine tests hopefully, finally, run. --- .vsts-ci.yml | 15 +++++++++------ documentation/project-docs/repro-helix-failure.md | 4 ++-- eng/pipelines/templates/jobs/sdk-build.yml | 7 +++---- test/UnitTests.proj | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b85e2240e073..7e8dd0e7bc20 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -170,8 +170,9 @@ extends: # This uses the build step for testing, so the extra test step is not necessary. runTests: false - categoryName: TemplateEngine - # Use backtick (`) to escape the semicolon for the PowerShell task. - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj`;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use %3B for semicolon so MSBuild can interpret it correctly. + # See: https://stackoverflow.com/a/44259937/294804 + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true @@ -302,8 +303,9 @@ extends: osProperties: /p:HostOSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) - # Use backslash (\) to escape the semicolon for the script task. - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj\;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use %3B for semicolon so MSBuild can interpret it correctly. + # See: https://stackoverflow.com/a/44259937/294804 + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -335,8 +337,9 @@ extends: ### TESTS ### - categoryName: TestBuild - categoryName: TemplateEngine - # Use backslash (\) to escape the semicolon for the script task. - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj\;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + # Use %3B for semicolon so MSBuild can interpret it correctly. + # See: https://stackoverflow.com/a/44259937/294804 + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true diff --git a/documentation/project-docs/repro-helix-failure.md b/documentation/project-docs/repro-helix-failure.md index 88826085e2f3..ffe972d79c17 100644 --- a/documentation/project-docs/repro-helix-failure.md +++ b/documentation/project-docs/repro-helix-failure.md @@ -18,7 +18,7 @@ REM use build env to have stage 0 dotnet on the PATH .\artifacts\sdk-build-env.bat REM run the special test target CreateLocalHelixTestLayout. To have the test layout created on disk. -dotnet msbuild /restore /t:CreateLocalHelixTestLayout .\test\UnitTests.proj /p:creator=dotnetsdkdev /p:_CustomHelixTargetQueue=Windows.Server.Amd64.VS2019.Pre.Open /bl +dotnet msbuild /restore /t:CreateLocalHelixTestLayout .\test\UnitTests.proj /p:creator=dotnetsdkdev /p:CustomHelixTargetQueue=Windows.Server.Amd64.VS2019.Pre.Open /bl ``` Copy the result of `artifacts\bin\localHelixTestLayout` to another directory or VM. For example to `C:\helix\localHelixTestLayout`. See "Folders under localHelixTestLayout" for the content. This is _correlation_ payload in helix term. @@ -64,4 +64,4 @@ Due to Helix long path problem. The folders under localHelixTestLayout all have - New test environment variable introduced has not been reflected in RunTestsOnHelix.cmd. -- To test out changes to the Helix tasks, you can run the unit test project directly: `dotnet -t:Test .\test\UnitTests.proj /p:_CustomHelixTargetQueue=foo /bl` It won't be able to push the files into helix but it will let you view what helix work items are getting created. +- To test out changes to the Helix tasks, you can run the unit test project directly: `dotnet -t:Test .\test\UnitTests.proj /p:CustomHelixTargetQueue=foo /bl` It won't be able to push the files into helix but it will let you view what helix work items are getting created. diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 40b98f54ffc9..c6ce33ca9698 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -109,17 +109,16 @@ jobs: - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - - script: $(Build.SourcesDirectory)/artifacts/bin/redist-installer/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update + - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload - ${{ if eq(parameters.pool.os, 'windows') }}: - # TODO: Change _CustomHelixTargetQueue name - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) -projects ${{ parameters.testProjects }} /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() @@ -137,7 +136,7 @@ jobs: /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() diff --git a/test/UnitTests.proj b/test/UnitTests.proj index 61872c2d7d3f..89de10dcfd0c 100644 --- a/test/UnitTests.proj +++ b/test/UnitTests.proj @@ -50,8 +50,8 @@ - - + + From c4c2b762981792abbb4bd0dc077ef1d749bf8b4f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 15 May 2024 11:21:13 -0700 Subject: [PATCH 65/90] Trying another workaround for specifying multiple projects. --- .vsts-ci.yml | 12 +++--------- eng/pipelines/templates/jobs/sdk-build.yml | 5 +++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7e8dd0e7bc20..e3c5534ee60a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -170,9 +170,7 @@ extends: # This uses the build step for testing, so the extra test step is not necessary. runTests: false - categoryName: TemplateEngine - # Use %3B for semicolon so MSBuild can interpret it correctly. - # See: https://stackoverflow.com/a/44259937/294804 - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true @@ -303,9 +301,7 @@ extends: osProperties: /p:HostOSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) - # Use %3B for semicolon so MSBuild can interpret it correctly. - # See: https://stackoverflow.com/a/44259937/294804 - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -337,9 +333,7 @@ extends: ### TESTS ### - categoryName: TestBuild - categoryName: TemplateEngine - # Use %3B for semicolon so MSBuild can interpret it correctly. - # See: https://stackoverflow.com/a/44259937/294804 - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj%3B$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index c6ce33ca9698..2645dc0b17d1 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -111,11 +111,12 @@ jobs: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload + # For the weird /p:Projects syntax, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) - -projects ${{ parameters.testProjects }} + '/p:Projects="${{ parameters.testProjects }}"' /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} @@ -131,7 +132,7 @@ jobs: - script: eng/common/build.sh -restore -test -ci -prepareMachine -configuration $(buildConfiguration) - -projects ${{ parameters.testProjects }} + '/p:Projects="${{ parameters.testProjects }}"' /p:Architecture=${{ parameters.buildArchitecture }} /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} From 5640710a8ed516d7e4c374313923a4a75536f057 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 15 May 2024 14:20:43 -0700 Subject: [PATCH 66/90] Trying workaround for PowerShell projects syntax. Trying to get alpine319WithNode tests to run. --- .vsts-ci.yml | 3 ++- eng/pipelines/templates/jobs/sdk-build.yml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e3c5534ee60a..b99a630c43ba 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -298,7 +298,8 @@ extends: container: alpine319WithNode runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. - osProperties: /p:HostOSName=linux-musl + # osProperties: /p:HostOSName=linux-musl + osProperties: /p:OSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 2645dc0b17d1..7dc0ef52fd7d 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -111,12 +111,12 @@ jobs: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update displayName: Install wasm-tools Workload - # For the weird /p:Projects syntax, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 + # For the /p:Projects syntax for PowerShell, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1146466335 - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) - '/p:Projects="${{ parameters.testProjects }}"' + /p:Projects=\`"${{ replace(parameters.testProjects, ';', '`;') }}\`" /p:Architecture=${{ parameters.buildArchitecture }} ${{ parameters.runtimeSourceProperties }} /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} @@ -129,6 +129,7 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) RunAoTTests: ${{ parameters.runAoTTests }} - ${{ else }}: + # For the /p:Projects syntax for Bash, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 - script: eng/common/build.sh -restore -test -ci -prepareMachine -configuration $(buildConfiguration) From cfa29b3db157a54a8044acfbc264b815fb17ea3d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 15 May 2024 15:52:09 -0700 Subject: [PATCH 67/90] Set alpine back to HostOSName. --- .vsts-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b99a630c43ba..e3c5534ee60a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -298,8 +298,7 @@ extends: container: alpine319WithNode runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. - # osProperties: /p:HostOSName=linux-musl - osProperties: /p:OSName=linux-musl + osProperties: /p:HostOSName=linux-musl - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj From 3754ba465d9c1ccf8063b96e9e59ae7ca832944f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 08:56:09 -0700 Subject: [PATCH 68/90] Adding container to the helix testing queue declaration. --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 7dc0ef52fd7d..54892022c0cf 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -138,7 +138,7 @@ jobs: /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} + /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }}@${{ parameters.container }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() From b9046e4e64675d7ba3c8d76063b381af9e678930 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 09:58:47 -0700 Subject: [PATCH 69/90] Converted the .vsts-pr.yml to use the new templates. Minor formatting adjustments. --- .vsts-ci.yml | 8 +- .vsts-pr.yml | 526 +++++++++++------- .../templates/jobs/sdk-job-matrix.yml | 1 + 3 files changed, 346 insertions(+), 189 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e3c5534ee60a..dca6dfa884f4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -66,9 +66,10 @@ extends: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode cblMariner20Fpm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - # WORKAROUND: The centosStream9 assets are not published. So, we need to revert to 8 for the time being. centosStream8: image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + centosStream9: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 debian11Amd64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 fedora39: @@ -289,6 +290,7 @@ extends: container: fedora39 osProperties: $(_linuxOsProperties) - categoryName: TestBuild + # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. container: centosStream8 osProperties: /p:OSName=linux - categoryName: TestBuild @@ -310,7 +312,7 @@ extends: name: Azure Pipelines image: macOS-latest os: macOS - helixTargetQueue: OSX.13.Amd64 + helixTargetQueue: osx.13.amd64 oneESCompat: templateFolderName: templates-official publishTaskPrefix: 1ES. @@ -342,7 +344,7 @@ extends: parameters: platform: name: Managed - container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + container: centosStream9 ############### DOTNET-FORMAT ############### - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 4610446a52e3..9e9a117fa1a4 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -10,198 +10,352 @@ pr: - internal/release/* variables: - - name: teamName - value: Roslyn-Project-System - - name: _CIBuild - value: -restore -build -sign -pack -ci - - name: _DotNetArtifactsCategory - value: .NETCore - - name: _DotNetValidationArtifactsCategory - value: .NETCore - # Disable post-build signing for internal release-branch builds or internal manual builds. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - - name: PostBuildSign - value: false - - ${{ else }}: - - name: PostBuildSign - value: true - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: _InternalRuntimeDownloadArgs - value: '' - - name: _OfficialBuildArgs - value: '' - - name: "skipComponentGovernanceDetection" - value: "true" - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - name: _OfficialBuildArgs - value: /p:OfficialBuilder=Microsoft - - name: Codeql.Enabled - value: true - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-CLI-SDLValidation-Params - - template: /eng/common/templates/variables/pool-providers.yml +- template: /eng/pipelines/templates/variables/sdk-defaults.yml +# Variables used: DncEngPublicBuildPool +- template: /eng/common/templates/variables/pool-providers.yml +### LOCAL ONLY ### +- name: _linuxOsProperties + value: /p:OSName=linux /p:IsLinuxPortable=true + +resources: + containers: + alpine319WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + centosStream8: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + centosStream9: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + debian11Amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + fedora39: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + ubuntu2204: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 stages: +############### BUILD STAGE ############### - stage: build displayName: Build jobs: - - job: Publish_Build_Configuration - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: 'windows-2019' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish Build Config - - template: /eng/build-pr.yml + ############### WINDOWS ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: - agentOs: Windows_NT pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals 1es-windows-2022-open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: test - _Test: -test - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: real - _Test: '' - - template: /eng/common/templates/job/source-build.yml + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals 1es-windows-2022-open + os: windows + helixTargetQueue: windows.amd64.vs2022.pre.open + publishRetryConfig: true + jobParameterSets: + ### TESTS ### + - categoryName: TestBuild + - categoryName: FullFramework + testFullMSBuild: true + - categoryName: TestAsTools + runTestsAsTool: true + # This uses the build step for testing, so the extra test step is not necessary. + runTests: false + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true + + ############### LINUX ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self parameters: - platform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - # Publish not needed for PR builds - skipPublishValidation: true - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build-pr.yml - parameters: - agentOs: Windows_NT_FullFramework - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals 1es-windows-2022-open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _PublishArgs: '' - _SignType: test - _Test: -test - - - template: /eng/build-pr.yml - parameters: - agentOs: Windows_NT_TestAsTools - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals 1es-windows-2022-open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _PublishArgs: '' - _SignType: test - - - template: /eng/build-pr.yml - parameters: - agentOs: Ubuntu_22_04 - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: 'ubuntu-22.04' - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals build.ubuntu.2204.amd64 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: ubuntu.2204.amd64.open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Ubuntu.2204.Amd64 - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - _SignType: test - _Test: -test - - - template: /eng/build-pr.yml - parameters: - agentOs: Darwin - pool: - vmImage: 'macOS-latest' - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.13.Amd64.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.13.Amd64 - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - _SignType: test - _Test: -test - - - template: /eng/template-engine.yml - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - dependsOn: - - Windows_NT - - Source_Build_Managed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/post-build/post-build.yml + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals build.ubuntu.2204.amd64.open + os: linux + helixTargetQueue: ubuntu.2204.amd64.open + jobParameterSets: + ### TESTS ### + - categoryName: TestBuild + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 + osProperties: $(_linuxOsProperties) + # Don't run the tests on arm64. Only perform the build itself. + runTests: false + - categoryName: TestBuild + container: ubuntu2204 + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + container: fedora39 + osProperties: $(_linuxOsProperties) + - categoryName: TestBuild + # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. + container: centosStream8 + osProperties: /p:OSName=linux + - categoryName: TestBuild + container: debian11Amd64 + osProperties: /p:OSName=linux /p:BuildSdkDeb=true + - categoryName: TestBuild + container: alpine319WithNode + runtimeIdentifier: linux-musl-x64 + # Use HostOSName when running on alpine. + osProperties: /p:HostOSName=linux-musl + - categoryName: TemplateEngine + osProperties: $(_linuxOsProperties) + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + + ############### MACOS ############### + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + parameters: + pool: + name: Azure Pipelines + demands: ImageOverride -equals macOS-latest + os: macOS + helixTargetQueue: osx.13.amd64.open + jobParameterSets: + ### TESTS ### + - categoryName: TestBuild + - categoryName: TemplateEngine + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj + - categoryName: AoT + runAoTTests: true + + ############### SOURCE BUILD ############### + - template: /eng/common/templates/job/source-build.yml@self parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "dotnet-sdk" - -TsaCodebaseName "dotnet-sdk" - -TsaPublish $True' + platform: + name: Managed + container: centosStream9 + + ############### DOTNET-FORMAT ############### + - template: /eng/dotnet-format/dotnet-format-integration.yml@self + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# variables: +# - name: teamName +# value: Roslyn-Project-System +# - name: _CIBuild +# value: -restore -build -sign -pack -ci +# - name: _DotNetArtifactsCategory +# value: .NETCore +# - name: _DotNetValidationArtifactsCategory +# value: .NETCore +# # Disable post-build signing for internal release-branch builds or internal manual builds. +# - ${{ if and(eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: +# - name: PostBuildSign +# value: false +# - ${{ else }}: +# - name: PostBuildSign +# value: true +# - ${{ if eq(variables['System.TeamProject'], 'public') }}: +# - name: _InternalRuntimeDownloadArgs +# value: '' +# - name: _OfficialBuildArgs +# value: '' +# - name: "skipComponentGovernanceDetection" +# value: "true" +# - ${{ if ne(variables['System.TeamProject'], 'public') }}: +# - name: _OfficialBuildArgs +# value: /p:OfficialBuilder=Microsoft +# - name: Codeql.Enabled +# value: true +# - group: DotNetBuilds storage account read tokens +# - name: _InternalRuntimeDownloadArgs +# value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal +# /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) +# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +# - group: DotNet-CLI-SDLValidation-Params +# - template: /eng/common/templates/variables/pool-providers.yml + +# stages: +# - stage: build +# displayName: Build +# jobs: +# - job: Publish_Build_Configuration +# pool: +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# vmImage: 'windows-2019' +# ${{ if eq(variables['System.TeamProject'], 'internal') }}: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2019.amd64 +# steps: +# - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration +# artifact: BuildConfiguration +# displayName: Publish Build Config +# - template: /eng/build-pr.yml +# parameters: +# agentOs: Windows_NT +# pool: +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngPublicBuildPool) +# demands: ImageOverride -equals 1es-windows-2022-open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals 1es-windows-2022 +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: Windows.Amd64.VS2022.Pre.Open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: Windows.Amd64.VS2022.Pre +# strategy: +# matrix: +# Build_Release: +# _BuildConfig: Release +# _PublishArgs: '' +# ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: +# _SignType: test +# _Test: -test +# ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +# _SignType: real +# _Test: '' +# - template: /eng/common/templates/job/source-build.yml +# parameters: +# platform: +# name: 'Managed' +# container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' +# # Publish not needed for PR builds +# skipPublishValidation: true +# - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: +# - template: /eng/build-pr.yml +# parameters: +# agentOs: Windows_NT_FullFramework +# pool: +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngPublicBuildPool) +# demands: ImageOverride -equals 1es-windows-2022-open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2022preview.amd64 +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: Windows.Amd64.VS2022.Pre.Open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: Windows.Amd64.VS2022.Pre +# strategy: +# matrix: +# Build_Debug: +# _BuildConfig: Debug +# _PublishArgs: '' +# _SignType: test +# _Test: -test + +# - template: /eng/build-pr.yml +# parameters: +# agentOs: Windows_NT_TestAsTools +# pool: +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngPublicBuildPool) +# demands: ImageOverride -equals 1es-windows-2022-open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals 1es-windows-2022 +# strategy: +# matrix: +# Build_Debug: +# _BuildConfig: Debug +# _PublishArgs: '' +# _SignType: test + +# - template: /eng/build-pr.yml +# parameters: +# agentOs: Ubuntu_22_04 +# pool: +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# vmImage: 'ubuntu-22.04' +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals build.ubuntu.2204.amd64 +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: ubuntu.2204.amd64.open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: Ubuntu.2204.Amd64 +# strategy: +# matrix: +# Build_Release: +# _BuildConfig: Release +# _PublishArgs: '' +# _SignType: test +# _Test: -test + +# - template: /eng/build-pr.yml +# parameters: +# agentOs: Darwin +# pool: +# vmImage: 'macOS-latest' +# ${{ if eq(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: OSX.13.Amd64.Open +# ${{ if ne(variables['System.TeamProject'], 'public') }}: +# helixTargetQueue: OSX.13.Amd64 +# strategy: +# matrix: +# Build_Release: +# _BuildConfig: Release +# _PublishArgs: '' +# _SignType: test +# _Test: -test + +# - template: /eng/template-engine.yml + +# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +# - template: /eng/common/templates/job/publish-build-assets.yml +# parameters: +# publishUsingPipelines: true +# publishAssetsImmediately: true +# dependsOn: +# - Windows_NT +# - Source_Build_Managed +# pool: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2019.amd64 +# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +# - template: /eng/common/templates/post-build/post-build.yml +# parameters: +# publishingInfraVersion: 3 +# enableSymbolValidation: false +# enableSigningValidation: false +# enableNugetValidation: false +# enableSourceLinkValidation: false +# publishInstallersAndChecksums: true +# publishAssetsImmediately: true +# SDLValidationParameters: +# enable: false +# params: ' -SourceToolsList @("policheck","credscan") +# -TsaInstanceURL $(_TsaInstanceURL) +# -TsaProjectName $(_TsaProjectName) +# -TsaNotificationEmail $(_TsaNotificationEmail) +# -TsaCodebaseAdmin $(_TsaCodebaseAdmin) +# -TsaBugAreaPath $(_TsaBugAreaPath) +# -TsaIterationPath $(_TsaIterationPath) +# -TsaRepositoryName "dotnet-sdk" +# -TsaCodebaseName "dotnet-sdk" +# -TsaPublish $True' diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 5e6756745f77..5844a56f3995 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -7,6 +7,7 @@ parameters: locBranch: '' jobs: +# TODO: Move this down a layer - ${{ if eq(parameters.publishRetryConfig, true) }}: # Publishes the build-configuration.json to the artifacts which Arcade uses to allow the jobs in the pipeline run to retry automatically. # https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/BuildRetryOnboard.md From b8224bf5a04243b7fd666edfaaae970453dd4ed9 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 10:03:27 -0700 Subject: [PATCH 70/90] Changed containers format from dictionary to list. --- .vsts-pr.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 9e9a117fa1a4..b735a40348a5 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -19,18 +19,18 @@ variables: resources: containers: - alpine319WithNode: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - centosStream8: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - centosStream9: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - debian11Amd64: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - fedora39: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - ubuntu2204: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + - container: alpine319WithNode + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + - container: centosStream8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + - container: centosStream9 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + - container: debian11Amd64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + - container: fedora39 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + - container: ubuntu2204 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 stages: ############### BUILD STAGE ############### From 1d3aea2b8a03791cd515d44b8ddbd66f4f6c8264 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 11:47:20 -0700 Subject: [PATCH 71/90] Added information to specify helix container for testing. Moved publishRetryConfig out of a separate job and just as a step during build. It is only specified for one job out of either official or test. Allowed the dotnet-format tests to run when specifying runTests. --- .vsts-ci.yml | 13 ++++++++++-- .vsts-pr.yml | 20 +++++++++++++------ eng/pipelines/templates/jobs/sdk-build.yml | 16 ++++++++++++++- .../templates/jobs/sdk-job-matrix.yml | 15 -------------- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index dca6dfa884f4..6e4020d9d4ef 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -47,6 +47,8 @@ variables: value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) - name: _linuxOsProperties value: /p:OSName=linux /p:IsLinuxPortable=true +- name: _helixTargetContainerPrefix + value: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' resources: repositories: @@ -106,7 +108,6 @@ extends: templateFolderName: templates-official publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - publishRetryConfig: true locBranch: release/8.0.4xx jobParameterSets: - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -116,6 +117,7 @@ extends: signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false + publishRetryConfig: true variables: _SignType: real - categoryName: Official @@ -164,6 +166,7 @@ extends: - ${{ else }}: ### TESTS ### - categoryName: TestBuild + publishRetryConfig: true - categoryName: FullFramework testFullMSBuild: true - categoryName: TestAsTools @@ -285,19 +288,25 @@ extends: runTests: false - categoryName: TestBuild container: ubuntu2204 + helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - categoryName: TestBuild container: fedora39 + # No fedora Helix container is available, so use the ubuntu one instead. + helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - categoryName: TestBuild # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. container: centosStream8 + helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - categoryName: TestBuild container: debian11Amd64 + helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - categoryName: TestBuild container: alpine319WithNode + helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl @@ -347,7 +356,7 @@ extends: container: centosStream9 ############### DOTNET-FORMAT ############### - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if or(eq(parameters.runTests, true), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/dotnet-format/dotnet-format-integration.yml@self parameters: oneESCompat: diff --git a/.vsts-pr.yml b/.vsts-pr.yml index b735a40348a5..6850b485d65b 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -16,6 +16,8 @@ variables: ### LOCAL ONLY ### - name: _linuxOsProperties value: /p:OSName=linux /p:IsLinuxPortable=true +- name: _helixTargetContainerPrefix + value: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' resources: containers: @@ -45,10 +47,10 @@ stages: demands: ImageOverride -equals 1es-windows-2022-open os: windows helixTargetQueue: windows.amd64.vs2022.pre.open - publishRetryConfig: true jobParameterSets: ### TESTS ### - categoryName: TestBuild + publishRetryConfig: true - categoryName: FullFramework testFullMSBuild: true - categoryName: TestAsTools @@ -61,7 +63,7 @@ stages: runAoTTests: true ############### LINUX ############### - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml parameters: pool: name: $(DncEngPublicBuildPool) @@ -80,19 +82,25 @@ stages: runTests: false - categoryName: TestBuild container: ubuntu2204 + helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - categoryName: TestBuild container: fedora39 + # No fedora Helix container is available, so use the ubuntu one instead. + helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - categoryName: TestBuild # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. container: centosStream8 + helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - categoryName: TestBuild container: debian11Amd64 + helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - categoryName: TestBuild container: alpine319WithNode + helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl @@ -101,11 +109,11 @@ stages: testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self + - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml parameters: pool: name: Azure Pipelines - demands: ImageOverride -equals macOS-latest + vmImage: macOS-latest os: macOS helixTargetQueue: osx.13.amd64.open jobParameterSets: @@ -117,14 +125,14 @@ stages: runAoTTests: true ############### SOURCE BUILD ############### - - template: /eng/common/templates/job/source-build.yml@self + - template: /eng/common/templates/job/source-build.yml parameters: platform: name: Managed container: centosStream9 ############### DOTNET-FORMAT ############### - - template: /eng/dotnet-format/dotnet-format-integration.yml@self + - template: /eng/dotnet-format/dotnet-format-integration.yml diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 54892022c0cf..a9c9b5fbb9c6 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -1,12 +1,15 @@ parameters: + ### GENERAL ### variables: {} oneESCompat: templateFolderName: templates publishTaskPrefix: '' container: '' + helixTargetContainer: '' categoryName: Build runTests: true testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj + publishRetryConfig: false ### ENV VARS ### testFullMSBuild: false runAoTTests: false @@ -44,6 +47,14 @@ jobs: steps: ############## PREP ############### + - ${{ if eq(parameters.publishRetryConfig, true) }}: + # Publishes the build-configuration.json to the artifacts which Arcade uses to allow all the jobs in the pipeline to retry automatically. + # See: https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/BuildRetryOnboard.md + - task: ${{ parameters.oneESCompat.publishTaskPrefix }}PublishBuildArtifacts@1 + displayName: Publish build-configuration.json + inputs: + PathtoPublish: $(Build.SourcesDirectory)/eng/BuildConfiguration + ArtifactName: BuildConfiguration - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token @@ -130,6 +141,9 @@ jobs: RunAoTTests: ${{ parameters.runAoTTests }} - ${{ else }}: # For the /p:Projects syntax for Bash, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 + # The /p:CustomHelixTargetQueue syntax is: ()@ + # The is optional and only used for logging purposes. + # For the Helix containers, see the 'simpleTags' arrays here: https://github.com/dotnet/versions/blob/main/build-info/docker/image-info.dotnet-dotnet-buildtools-prereqs-docker-main.json - script: eng/common/build.sh -restore -test -ci -prepareMachine -configuration $(buildConfiguration) @@ -138,7 +152,7 @@ jobs: /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }}@${{ parameters.container }} + /p:CustomHelixTargetQueue=($(System.PhaseName))${{ parameters.helixTargetQueue }}${{ parameters.helixTargetContainer }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 5844a56f3995..58a0e4f1be48 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -2,24 +2,9 @@ parameters: jobParameterSets: [] oneESCompat: templateFolderName: templates - publishTaskPrefix: '' - publishRetryConfig: false locBranch: '' jobs: -# TODO: Move this down a layer -- ${{ if eq(parameters.publishRetryConfig, true) }}: - # Publishes the build-configuration.json to the artifacts which Arcade uses to allow the jobs in the pipeline run to retry automatically. - # https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/BuildRetryOnboard.md - - job: PublishRetryConfig - displayName: Publish Pipeline Retry Config - pool: ${{ parameters.pool }} - steps: - - task: ${{ parameters.oneESCompat.publishTaskPrefix }}PublishBuildArtifacts@1 - displayName: Publish build-configuration.json - inputs: - PathtoPublish: $(Build.SourcesDirectory)/eng/BuildConfiguration - ArtifactName: BuildConfiguration - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/job/onelocbuild.yml parameters: From 3e02e40ef8c224de717a9a630a07454f5ccad5ec Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 12:06:07 -0700 Subject: [PATCH 72/90] Trying to escape parentheses for the helix queue param. --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index a9c9b5fbb9c6..d4087d8b60f9 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -152,7 +152,7 @@ jobs: /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - /p:CustomHelixTargetQueue=($(System.PhaseName))${{ parameters.helixTargetQueue }}${{ parameters.helixTargetContainer }} + '/p:CustomHelixTargetQueue="($(System.PhaseName))${{ parameters.helixTargetQueue }}${{ parameters.helixTargetContainer }}"' /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() From 3891837a20924e8ab901089847eba946b0ea62ee Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 14:49:21 -0700 Subject: [PATCH 73/90] Fix CustomHelixTargetQueue formatting. Disable tests for all container-based builds. --- .vsts-ci.yml | 20 +++++++++++++++----- .vsts-pr.yml | 20 +++++++++++++++----- eng/pipelines/templates/jobs/sdk-build.yml | 11 ++--------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6e4020d9d4ef..90810106a191 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -286,30 +286,40 @@ extends: osProperties: $(_linuxOsProperties) # Don't run the tests on arm64. Only perform the build itself. runTests: false - - categoryName: TestBuild + - categoryName: ContainerBased container: ubuntu2204 helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: fedora39 # No fedora Helix container is available, so use the ubuntu one instead. helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. container: centosStream8 helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: debian11Amd64 helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: alpine319WithNode helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl + # WORKAROUND: Skipping all container-based testing for now. + runTests: false - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 6850b485d65b..fc751ac8a1d5 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -80,30 +80,40 @@ stages: osProperties: $(_linuxOsProperties) # Don't run the tests on arm64. Only perform the build itself. runTests: false - - categoryName: TestBuild + - categoryName: ContainerBased container: ubuntu2204 helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: fedora39 # No fedora Helix container is available, so use the ubuntu one instead. helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. container: centosStream8 helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: debian11Amd64 helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - - categoryName: TestBuild + # WORKAROUND: Skipping all container-based testing for now. + runTests: false + - categoryName: ContainerBased container: alpine319WithNode helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl + # WORKAROUND: Skipping all container-based testing for now. + runTests: false - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index d4087d8b60f9..b389e807938c 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -83,9 +83,6 @@ jobs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: - # Required by Arcade for passing to Helix - # SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - ${{ else }}: @@ -110,9 +107,6 @@ jobs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) displayName: Build env: - # Required by Arcade for passing to Helix - # SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # HelixAccessToken: $(HelixApiAccessToken) BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} @@ -141,8 +135,7 @@ jobs: RunAoTTests: ${{ parameters.runAoTTests }} - ${{ else }}: # For the /p:Projects syntax for Bash, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 - # The /p:CustomHelixTargetQueue syntax is: ()@ - # The is optional and only used for logging purposes. + # The /p:CustomHelixTargetQueue syntax is: @ # For the Helix containers, see the 'simpleTags' arrays here: https://github.com/dotnet/versions/blob/main/build-info/docker/image-info.dotnet-dotnet-buildtools-prereqs-docker-main.json - script: eng/common/build.sh -restore -test -ci -prepareMachine @@ -152,7 +145,7 @@ jobs: /p:Rid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} ${{ parameters.runtimeSourceProperties }} - '/p:CustomHelixTargetQueue="($(System.PhaseName))${{ parameters.helixTargetQueue }}${{ parameters.helixTargetContainer }}"' + /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }}${{ parameters.helixTargetContainer }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/${{ parameters.categoryName }}Tests.binlog displayName: Run ${{ parameters.categoryName }} Tests condition: succeeded() From dfb0876889551099a864635a770be31aac749827 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 16 May 2024 16:22:52 -0700 Subject: [PATCH 74/90] Fixed running publish only for non-test runs. Cleaned up commented code. Added issue links to comments with workarounds. Made the component governance skip only for test builds. --- .vsts-ci.yml | 27 ++++-- .vsts-pr.yml | 254 +++------------------------------------------------ 2 files changed, 30 insertions(+), 251 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 90810106a191..1c87d2587f54 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -87,9 +87,10 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - componentgovernance: - # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - ignoreDirectories: artifacts, .packages + ${{ if or(eq(parameters.runTests, true), eq(variables['Build.Reason'], 'PullRequest')) }}: + componentgovernance: + # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ + ignoreDirectories: artifacts, .packages stages: ############### BUILD STAGE ############### @@ -290,27 +291,32 @@ extends: container: ubuntu2204 helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: fedora39 # No fedora Helix container is available, so use the ubuntu one instead. helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased - # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. + # The centosStream9 assets are not published. So, we need to use 8 for the time being. + # See: https://github.com/dotnet/runtime/issues/102344 container: centosStream8 helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: debian11Amd64 helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: alpine319WithNode @@ -318,7 +324,8 @@ extends: runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) @@ -373,7 +380,7 @@ extends: publishTaskPrefix: 1ES. ############### PUBLISH STAGE ############### - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - stage: publish displayName: Publish dependsOn: build diff --git a/.vsts-pr.yml b/.vsts-pr.yml index fc751ac8a1d5..9e74a6e34148 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -84,27 +84,32 @@ stages: container: ubuntu2204 helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: fedora39 # No fedora Helix container is available, so use the ubuntu one instead. helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 osProperties: $(_linuxOsProperties) - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased - # WORKAROUND: The centosStream9 assets are not published. So, we need to use 8 for the time being. + # The centosStream9 assets are not published. So, we need to use 8 for the time being. + # See: https://github.com/dotnet/runtime/issues/102344 container: centosStream8 helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix osProperties: /p:OSName=linux - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: debian11Amd64 helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: ContainerBased container: alpine319WithNode @@ -112,7 +117,8 @@ stages: runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl - # WORKAROUND: Skipping all container-based testing for now. + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 runTests: false - categoryName: TemplateEngine osProperties: $(_linuxOsProperties) @@ -142,238 +148,4 @@ stages: container: centosStream9 ############### DOTNET-FORMAT ############### - - template: /eng/dotnet-format/dotnet-format-integration.yml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# variables: -# - name: teamName -# value: Roslyn-Project-System -# - name: _CIBuild -# value: -restore -build -sign -pack -ci -# - name: _DotNetArtifactsCategory -# value: .NETCore -# - name: _DotNetValidationArtifactsCategory -# value: .NETCore -# # Disable post-build signing for internal release-branch builds or internal manual builds. -# - ${{ if and(eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: -# - name: PostBuildSign -# value: false -# - ${{ else }}: -# - name: PostBuildSign -# value: true -# - ${{ if eq(variables['System.TeamProject'], 'public') }}: -# - name: _InternalRuntimeDownloadArgs -# value: '' -# - name: _OfficialBuildArgs -# value: '' -# - name: "skipComponentGovernanceDetection" -# value: "true" -# - ${{ if ne(variables['System.TeamProject'], 'public') }}: -# - name: _OfficialBuildArgs -# value: /p:OfficialBuilder=Microsoft -# - name: Codeql.Enabled -# value: true -# - group: DotNetBuilds storage account read tokens -# - name: _InternalRuntimeDownloadArgs -# value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal -# /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) -# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: -# - group: DotNet-CLI-SDLValidation-Params -# - template: /eng/common/templates/variables/pool-providers.yml - -# stages: -# - stage: build -# displayName: Build -# jobs: -# - job: Publish_Build_Configuration -# pool: -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# vmImage: 'windows-2019' -# ${{ if eq(variables['System.TeamProject'], 'internal') }}: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 -# steps: -# - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration -# artifact: BuildConfiguration -# displayName: Publish Build Config -# - template: /eng/build-pr.yml -# parameters: -# agentOs: Windows_NT -# pool: -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngPublicBuildPool) -# demands: ImageOverride -equals 1es-windows-2022-open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals 1es-windows-2022 -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: Windows.Amd64.VS2022.Pre.Open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: Windows.Amd64.VS2022.Pre -# strategy: -# matrix: -# Build_Release: -# _BuildConfig: Release -# _PublishArgs: '' -# ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: -# _SignType: test -# _Test: -test -# ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: -# _SignType: real -# _Test: '' -# - template: /eng/common/templates/job/source-build.yml -# parameters: -# platform: -# name: 'Managed' -# container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' -# # Publish not needed for PR builds -# skipPublishValidation: true -# - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: -# - template: /eng/build-pr.yml -# parameters: -# agentOs: Windows_NT_FullFramework -# pool: -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngPublicBuildPool) -# demands: ImageOverride -equals 1es-windows-2022-open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2022preview.amd64 -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: Windows.Amd64.VS2022.Pre.Open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: Windows.Amd64.VS2022.Pre -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _PublishArgs: '' -# _SignType: test -# _Test: -test - -# - template: /eng/build-pr.yml -# parameters: -# agentOs: Windows_NT_TestAsTools -# pool: -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngPublicBuildPool) -# demands: ImageOverride -equals 1es-windows-2022-open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals 1es-windows-2022 -# strategy: -# matrix: -# Build_Debug: -# _BuildConfig: Debug -# _PublishArgs: '' -# _SignType: test - -# - template: /eng/build-pr.yml -# parameters: -# agentOs: Ubuntu_22_04 -# pool: -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# vmImage: 'ubuntu-22.04' -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals build.ubuntu.2204.amd64 -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: ubuntu.2204.amd64.open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: Ubuntu.2204.Amd64 -# strategy: -# matrix: -# Build_Release: -# _BuildConfig: Release -# _PublishArgs: '' -# _SignType: test -# _Test: -test - -# - template: /eng/build-pr.yml -# parameters: -# agentOs: Darwin -# pool: -# vmImage: 'macOS-latest' -# ${{ if eq(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: OSX.13.Amd64.Open -# ${{ if ne(variables['System.TeamProject'], 'public') }}: -# helixTargetQueue: OSX.13.Amd64 -# strategy: -# matrix: -# Build_Release: -# _BuildConfig: Release -# _PublishArgs: '' -# _SignType: test -# _Test: -test - -# - template: /eng/template-engine.yml - -# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: -# - template: /eng/common/templates/job/publish-build-assets.yml -# parameters: -# publishUsingPipelines: true -# publishAssetsImmediately: true -# dependsOn: -# - Windows_NT -# - Source_Build_Managed -# pool: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 -# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: -# - template: /eng/common/templates/post-build/post-build.yml -# parameters: -# publishingInfraVersion: 3 -# enableSymbolValidation: false -# enableSigningValidation: false -# enableNugetValidation: false -# enableSourceLinkValidation: false -# publishInstallersAndChecksums: true -# publishAssetsImmediately: true -# SDLValidationParameters: -# enable: false -# params: ' -SourceToolsList @("policheck","credscan") -# -TsaInstanceURL $(_TsaInstanceURL) -# -TsaProjectName $(_TsaProjectName) -# -TsaNotificationEmail $(_TsaNotificationEmail) -# -TsaCodebaseAdmin $(_TsaCodebaseAdmin) -# -TsaBugAreaPath $(_TsaBugAreaPath) -# -TsaIterationPath $(_TsaIterationPath) -# -TsaRepositoryName "dotnet-sdk" -# -TsaCodebaseName "dotnet-sdk" -# -TsaPublish $True' + - template: /eng/dotnet-format/dotnet-format-integration.yml \ No newline at end of file From d1783b7ba267fb7e016d6501d501346f582f7638 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 09:39:17 -0700 Subject: [PATCH 75/90] Disable SBOM for alpine jobs. --- .vsts-ci.yml | 4 ++++ .vsts-pr.yml | 2 ++ eng/pipelines/templates/jobs/sdk-build.yml | 2 ++ eng/pipelines/templates/jobs/sdk-job-matrix.yml | 4 ++++ 4 files changed, 12 insertions(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1c87d2587f54..f317391270a7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -246,6 +246,8 @@ extends: officialBuildProperties: $(_officialBuildProperties) # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl + # Sbom generation is not supported for alpine. + enableSbom: false runTests: false - categoryName: Musl container: mariner20CrossArm @@ -324,6 +326,8 @@ extends: runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl + # Sbom generation is not supported for alpine. + enableSbom: false # Skipping all container-based testing for now. # See: https://github.com/dotnet/sdk/issues/40935 runTests: false diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 9e74a6e34148..2d3f1574bf75 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -117,6 +117,8 @@ stages: runtimeIdentifier: linux-musl-x64 # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl + # Sbom generation is not supported for alpine. + enableSbom: false # Skipping all container-based testing for now. # See: https://github.com/dotnet/sdk/issues/40935 runTests: false diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index b389e807938c..8ad806f96b39 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -10,6 +10,7 @@ parameters: runTests: true testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj publishRetryConfig: false + enableSbom: true ### ENV VARS ### testFullMSBuild: false runAoTTests: false @@ -42,6 +43,7 @@ jobs: enablePublishBuildAssets: true enableTelemetry: true enablePublishUsingPipelines: true + enableSbom: ${{ parameters.enableSbom }} variables: - ${{ insert }}: ${{ parameters.variables }} diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 58a0e4f1be48..b03e8d31f08b 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -5,6 +5,7 @@ parameters: locBranch: '' jobs: +### ONELOCBUILD ### - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.locBranch }}')) }}: - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/job/onelocbuild.yml parameters: @@ -13,6 +14,9 @@ jobs: LclPackageId: LCL-JUNO-PROD-DOTNETSDK MirrorBranch: ${{ parameters.locBranch }} MirrorRepo: sdk +### BUILD JOBS ### +# Each job parameter set creates a new job and the parameter set is unwrapped into the parameters for that job. +# The parameters sent to this template are also passed to the job, which does include the jobParameterSets array, but that array itself isn't used in the job. - ${{ each jobParameters in parameters.jobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: From fd61dda8458c81e5c80c510e8e52fb54e4cb0eed Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 10:28:54 -0700 Subject: [PATCH 76/90] Made sdk-job-matrix.yml have the default parameter sets per OS. Made the CI pipeline pass in the parameter set per OS. Moved some reused values to sdk-defaults.yml. --- .vsts-ci.yml | 110 +++--------------- .vsts-pr.yml | 81 ------------- .../templates/jobs/sdk-job-matrix.yml | 102 ++++++++++++++-- .../templates/variables/sdk-defaults.yml | 3 + 4 files changed, 113 insertions(+), 183 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index f317391270a7..4786ad647100 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -45,10 +45,6 @@ variables: value: -sign /p:SignCoreSdk=true - name: _officialBuildProperties value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber) -- name: _linuxOsProperties - value: /p:OSName=linux /p:IsLinuxPortable=true -- name: _helixTargetContainerPrefix - value: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' resources: repositories: @@ -110,8 +106,8 @@ extends: publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) locBranch: release/8.0.4xx - jobParameterSets: - - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + windowsJobParameterSets: ### OFFICIAL ### - categoryName: Official publishArgument: $(_publishArgument) /p:EnableToolsetPublish=true @@ -164,20 +160,6 @@ extends: runTests: false variables: _SignType: real - - ${{ else }}: - ### TESTS ### - - categoryName: TestBuild - publishRetryConfig: true - - categoryName: FullFramework - testFullMSBuild: true - - categoryName: TestAsTools - runTestsAsTool: true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true ############### LINUX ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -191,28 +173,28 @@ extends: templateFolderName: templates-official publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - jobParameterSets: - - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + linuxJobParameterSets: ### OFFICIAL ### # Note: These builds are also portable like the Portable category, but that category uses containers, and doesn't publish zips and tarballs. - categoryName: Official publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) + osProperties: $(linuxOsPortableProperties) runTests: false - categoryName: Official buildArchitecture: arm runtimeIdentifier: linux-arm publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) + osProperties: $(linuxOsPortableProperties) runTests: false - categoryName: Official buildArchitecture: arm64 runtimeIdentifier: linux-arm64 publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) + osProperties: $(linuxOsPortableProperties) runTests: false ### PORTABLE ### - categoryName: Portable @@ -220,14 +202,14 @@ extends: # Do not publish zips and tarballs. The linux-x64 binaries are already published by Official. publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) /p:BuildSdkDeb=true + osProperties: $(linuxOsPortableProperties) /p:BuildSdkDeb=true runTests: false - categoryName: Portable container: cblMariner20Fpm # Do not publish zips and tarballs. The linux-x64 binaries are already published by Official. publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) /p:IsRPMBasedDistro=true + osProperties: $(linuxOsPortableProperties) /p:IsRPMBasedDistro=true runTests: false - categoryName: Portable container: cblMariner20Fpm @@ -236,7 +218,7 @@ extends: # Do not publish zips and tarballs. The linux-arm64 binaries are already published by Official. publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) /p:IsRPMBasedDistro=true + osProperties: $(linuxOsPortableProperties) /p:IsRPMBasedDistro=true runTests: false ### MUSL ### - categoryName: Musl @@ -246,7 +228,7 @@ extends: officialBuildProperties: $(_officialBuildProperties) # Use HostOSName when running on alpine. osProperties: /p:HostOSName=linux-musl - # Sbom generation is not supported for alpine. + # SBOM generation is not supported for alpine. enableSbom: false runTests: false - categoryName: Musl @@ -269,7 +251,7 @@ extends: pgoInstrument: true publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) + osProperties: $(linuxOsPortableProperties) runTests: false - categoryName: PGO pgoInstrument: true @@ -277,63 +259,8 @@ extends: runtimeIdentifier: linux-arm64 publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) - osProperties: $(_linuxOsProperties) - runTests: false - - ${{ else }}: - ### TESTS ### - - categoryName: TestBuild - osProperties: $(_linuxOsProperties) - - categoryName: TestBuild - buildArchitecture: arm64 - runtimeIdentifier: linux-arm64 - osProperties: $(_linuxOsProperties) - # Don't run the tests on arm64. Only perform the build itself. - runTests: false - - categoryName: ContainerBased - container: ubuntu2204 - helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 - osProperties: $(_linuxOsProperties) - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: fedora39 - # No fedora Helix container is available, so use the ubuntu one instead. - helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 - osProperties: $(_linuxOsProperties) - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - # The centosStream9 assets are not published. So, we need to use 8 for the time being. - # See: https://github.com/dotnet/runtime/issues/102344 - container: centosStream8 - helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix - osProperties: /p:OSName=linux - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: debian11Amd64 - helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 - osProperties: /p:OSName=linux /p:BuildSdkDeb=true - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: alpine319WithNode - helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 - runtimeIdentifier: linux-musl-x64 - # Use HostOSName when running on alpine. - osProperties: /p:HostOSName=linux-musl - # Sbom generation is not supported for alpine. - enableSbom: false - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 + osProperties: $(linuxOsPortableProperties) runTests: false - - categoryName: TemplateEngine - osProperties: $(_linuxOsProperties) - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -347,8 +274,8 @@ extends: templateFolderName: templates-official publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - jobParameterSets: - - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + macOSJobParameterSets: ### OFFICIAL ### - categoryName: Official runtimeIdentifier: osx-x64 @@ -361,13 +288,6 @@ extends: publishArgument: $(_publishArgument) officialBuildProperties: $(_officialBuildProperties) runTests: false - - ${{ else }}: - ### TESTS ### - - categoryName: TestBuild - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true ############### SOURCE BUILD ############### - template: /eng/common/templates-official/job/source-build.yml@self diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 2d3f1574bf75..af218304162f 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -13,11 +13,6 @@ variables: - template: /eng/pipelines/templates/variables/sdk-defaults.yml # Variables used: DncEngPublicBuildPool - template: /eng/common/templates/variables/pool-providers.yml -### LOCAL ONLY ### -- name: _linuxOsProperties - value: /p:OSName=linux /p:IsLinuxPortable=true -- name: _helixTargetContainerPrefix - value: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' resources: containers: @@ -47,20 +42,6 @@ stages: demands: ImageOverride -equals 1es-windows-2022-open os: windows helixTargetQueue: windows.amd64.vs2022.pre.open - jobParameterSets: - ### TESTS ### - - categoryName: TestBuild - publishRetryConfig: true - - categoryName: FullFramework - testFullMSBuild: true - - categoryName: TestAsTools - runTestsAsTool: true - # This uses the build step for testing, so the extra test step is not necessary. - runTests: false - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true ############### LINUX ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -70,61 +51,6 @@ stages: demands: ImageOverride -equals build.ubuntu.2204.amd64.open os: linux helixTargetQueue: ubuntu.2204.amd64.open - jobParameterSets: - ### TESTS ### - - categoryName: TestBuild - osProperties: $(_linuxOsProperties) - - categoryName: TestBuild - buildArchitecture: arm64 - runtimeIdentifier: linux-arm64 - osProperties: $(_linuxOsProperties) - # Don't run the tests on arm64. Only perform the build itself. - runTests: false - - categoryName: ContainerBased - container: ubuntu2204 - helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 - osProperties: $(_linuxOsProperties) - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: fedora39 - # No fedora Helix container is available, so use the ubuntu one instead. - helixTargetContainer: $(_helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 - osProperties: $(_linuxOsProperties) - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - # The centosStream9 assets are not published. So, we need to use 8 for the time being. - # See: https://github.com/dotnet/runtime/issues/102344 - container: centosStream8 - helixTargetContainer: $(_helixTargetContainerPrefix)centos-stream8-helix - osProperties: /p:OSName=linux - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: debian11Amd64 - helixTargetContainer: $(_helixTargetContainerPrefix)debian-11-helix-amd64 - osProperties: /p:OSName=linux /p:BuildSdkDeb=true - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: ContainerBased - container: alpine319WithNode - helixTargetContainer: $(_helixTargetContainerPrefix)alpine-3.18-helix-amd64 - runtimeIdentifier: linux-musl-x64 - # Use HostOSName when running on alpine. - osProperties: /p:HostOSName=linux-musl - # Sbom generation is not supported for alpine. - enableSbom: false - # Skipping all container-based testing for now. - # See: https://github.com/dotnet/sdk/issues/40935 - runTests: false - - categoryName: TemplateEngine - osProperties: $(_linuxOsProperties) - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### MACOS ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -134,13 +60,6 @@ stages: vmImage: macOS-latest os: macOS helixTargetQueue: osx.13.amd64.open - jobParameterSets: - ### TESTS ### - - categoryName: TestBuild - - categoryName: TemplateEngine - testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - - categoryName: AoT - runAoTTests: true ############### SOURCE BUILD ############### - template: /eng/common/templates/job/source-build.yml diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index b03e8d31f08b..4931892f9591 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -1,8 +1,83 @@ parameters: - jobParameterSets: [] oneESCompat: templateFolderName: templates locBranch: '' + ### WINDOWS ### + windowsJobParameterSets: + - categoryName: TestBuild + publishRetryConfig: true + - categoryName: FullFramework + testFullMSBuild: true + - categoryName: TestAsTools + runTestsAsTool: true + # This job uses the build step for testing, so the extra test step is not necessary. + runTests: false + - categoryName: TemplateEngine + testProjects: $(templateEngineTestProjects) + - categoryName: AoT + runAoTTests: true + ### MACOS ### + macOSJobParameterSets: + - categoryName: TestBuild + - categoryName: TemplateEngine + testProjects: $(templateEngineTestProjects) + - categoryName: AoT + runAoTTests: true + ### LINUX ### + linuxJobParameterSets: + - categoryName: TestBuild + osProperties: $(linuxOsPortableProperties) + - categoryName: TestBuild + buildArchitecture: arm64 + runtimeIdentifier: linux-arm64 + osProperties: $(linuxOsPortableProperties) + # Don't run the tests on arm64. Only perform the build itself. + runTests: false + - categoryName: ContainerBased + container: ubuntu2204 + helixTargetContainer: $(helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 + osProperties: $(linuxOsPortableProperties) + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 + runTests: false + - categoryName: ContainerBased + container: fedora39 + # No fedora Helix container is available, so use the ubuntu one instead. + helixTargetContainer: $(helixTargetContainerPrefix)ubuntu-22.04-helix-amd64 + osProperties: $(linuxOsPortableProperties) + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 + runTests: false + - categoryName: ContainerBased + # The centosStream9 assets are not published. So, we need to use 8 for the time being. + # See: https://github.com/dotnet/runtime/issues/102344 + container: centosStream8 + helixTargetContainer: $(helixTargetContainerPrefix)centos-stream8-helix + osProperties: /p:OSName=linux + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 + runTests: false + - categoryName: ContainerBased + container: debian11Amd64 + helixTargetContainer: $(helixTargetContainerPrefix)debian-11-helix-amd64 + osProperties: /p:OSName=linux /p:BuildSdkDeb=true + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 + runTests: false + - categoryName: ContainerBased + container: alpine319WithNode + helixTargetContainer: $(helixTargetContainerPrefix)alpine-3.18-helix-amd64 + runtimeIdentifier: linux-musl-x64 + # Use HostOSName when running on alpine. + osProperties: /p:HostOSName=linux-musl + # SBOM generation is not supported for alpine. + enableSbom: false + # Skipping all container-based testing for now. + # See: https://github.com/dotnet/sdk/issues/40935 + runTests: false + - categoryName: TemplateEngine + osProperties: $(linuxOsPortableProperties) + testProjects: $(templateEngineTestProjects) jobs: ### ONELOCBUILD ### @@ -16,9 +91,22 @@ jobs: MirrorRepo: sdk ### BUILD JOBS ### # Each job parameter set creates a new job and the parameter set is unwrapped into the parameters for that job. -# The parameters sent to this template are also passed to the job, which does include the jobParameterSets array, but that array itself isn't used in the job. -- ${{ each jobParameters in parameters.jobParameterSets }}: - - template: /eng/pipelines/templates/jobs/sdk-build.yml - parameters: - ${{ insert }}: ${{ parameters }} - ${{ insert }}: ${{ jobParameters }} \ No newline at end of file +# The parameters sent to this template are also passed to the job, which does include the os-specific jobParameterSets array, but that array itself isn't used in the job. +- ${{ if eq(parameters.pool.os, 'windows') }}: + - ${{ each jobParameters in parameters.windowsJobParameterSets }}: + - template: /eng/pipelines/templates/jobs/sdk-build.yml + parameters: + ${{ insert }}: ${{ parameters }} + ${{ insert }}: ${{ jobParameters }} +- ${{ if eq(parameters.pool.os, 'macOS') }}: + - ${{ each jobParameters in parameters.macOSJobParameterSets }}: + - template: /eng/pipelines/templates/jobs/sdk-build.yml + parameters: + ${{ insert }}: ${{ parameters }} + ${{ insert }}: ${{ jobParameters }} +- ${{ else }}: + - ${{ each jobParameters in parameters.linuxJobParameterSets }}: + - template: /eng/pipelines/templates/jobs/sdk-build.yml + parameters: + ${{ insert }}: ${{ parameters }} + ${{ insert }}: ${{ jobParameters }} \ No newline at end of file diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 7f7a0b5d2d61..803330dcb2a5 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -1,6 +1,9 @@ variables: ############### BUILD ############### buildConfiguration: Release + linuxOsPortableProperties: /p:OSName=linux /p:IsLinuxPortable=true + helixTargetContainerPrefix: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' + templateEngineTestProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### ARCADE ############### # "Post-build signing" means that this build will NOT sign itself. From 16ca606189f1e3981d7c00805da0d5cfdbaa4205 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 10:40:44 -0700 Subject: [PATCH 77/90] Realized the else condition won't work. Instead, made each explicit as windows, linux, and macOS to match the rest of the design. --- .../templates/jobs/sdk-job-matrix.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 4931892f9591..9c6236d731ca 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -12,13 +12,6 @@ parameters: runTestsAsTool: true # This job uses the build step for testing, so the extra test step is not necessary. runTests: false - - categoryName: TemplateEngine - testProjects: $(templateEngineTestProjects) - - categoryName: AoT - runAoTTests: true - ### MACOS ### - macOSJobParameterSets: - - categoryName: TestBuild - categoryName: TemplateEngine testProjects: $(templateEngineTestProjects) - categoryName: AoT @@ -78,6 +71,13 @@ parameters: - categoryName: TemplateEngine osProperties: $(linuxOsPortableProperties) testProjects: $(templateEngineTestProjects) + ### MACOS ### + macOSJobParameterSets: + - categoryName: TestBuild + - categoryName: TemplateEngine + testProjects: $(templateEngineTestProjects) + - categoryName: AoT + runAoTTests: true jobs: ### ONELOCBUILD ### @@ -98,14 +98,14 @@ jobs: parameters: ${{ insert }}: ${{ parameters }} ${{ insert }}: ${{ jobParameters }} -- ${{ if eq(parameters.pool.os, 'macOS') }}: - - ${{ each jobParameters in parameters.macOSJobParameterSets }}: +- ${{ if eq(parameters.pool.os, 'linux') }}: + - ${{ each jobParameters in parameters.linuxJobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: ${{ insert }}: ${{ parameters }} ${{ insert }}: ${{ jobParameters }} -- ${{ else }}: - - ${{ each jobParameters in parameters.linuxJobParameterSets }}: +- ${{ if eq(parameters.pool.os, 'macOS') }}: + - ${{ each jobParameters in parameters.macOSJobParameterSets }}: - template: /eng/pipelines/templates/jobs/sdk-build.yml parameters: ${{ insert }}: ${{ parameters }} From a7b6c48b8f986664d4bac6645461b8ecd4094ad7 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 11:14:04 -0700 Subject: [PATCH 78/90] Changed the CI parameter runTests to runTestBuild to not be confused with the runTests of the sdk-build template. --- .vsts-ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 4786ad647100..185400af6ce6 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -17,8 +17,9 @@ pr: - internal/release/* parameters: -- name: runTests - displayName: Run Tests +# When true, runs the pipeline in the same way as the PR pipeline. +- name: runTestBuild + displayName: Run A Test Build type: boolean default: false @@ -32,7 +33,7 @@ variables: # Variables used: HelixApiAccessToken - group: DotNet-HelixApi-Access # Allows Arcade to run a signed build by disabling post-build signing for release-branch builds or manual builds that are not running tests. -- ${{ if and(eq(parameters.runTests, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: +- ${{ if and(eq(parameters.runTestBuild, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}: - name: PostBuildSign value: false # Provides TSA variables for automatic bug reporting. @@ -83,7 +84,7 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - ${{ if or(eq(parameters.runTests, true), eq(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if or(eq(parameters.runTestBuild, true), eq(variables['Build.Reason'], 'PullRequest')) }}: componentgovernance: # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ ignoreDirectories: artifacts, .packages @@ -106,7 +107,7 @@ extends: publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) locBranch: release/8.0.4xx - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: windowsJobParameterSets: ### OFFICIAL ### - categoryName: Official @@ -173,7 +174,7 @@ extends: templateFolderName: templates-official publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: linuxJobParameterSets: ### OFFICIAL ### # Note: These builds are also portable like the Portable category, but that category uses containers, and doesn't publish zips and tarballs. @@ -274,7 +275,7 @@ extends: templateFolderName: templates-official publishTaskPrefix: 1ES. runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: macOSJobParameterSets: ### OFFICIAL ### - categoryName: Official @@ -297,14 +298,14 @@ extends: container: centosStream9 ############### DOTNET-FORMAT ############### - - ${{ if or(eq(parameters.runTests, true), eq(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.runTestBuild, true), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/dotnet-format/dotnet-format-integration.yml@self parameters: oneESCompat: publishTaskPrefix: 1ES. ############### PUBLISH STAGE ############### - - ${{ if and(eq(parameters.runTests, false), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: - stage: publish displayName: Publish dependsOn: build From af0112ba2c2f7f8b5ad6c387b1764df4439a7c30 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 11:31:45 -0700 Subject: [PATCH 79/90] Realized you cannot make the template engine testProjects a variable as there is a compile-time replacement expression that is necessary for windows powershell. --- eng/pipelines/templates/jobs/sdk-job-matrix.yml | 6 +++--- eng/pipelines/templates/variables/sdk-defaults.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 9c6236d731ca..6ce01ec94579 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -13,7 +13,7 @@ parameters: # This job uses the build step for testing, so the extra test step is not necessary. runTests: false - categoryName: TemplateEngine - testProjects: $(templateEngineTestProjects) + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true ### LINUX ### @@ -70,12 +70,12 @@ parameters: runTests: false - categoryName: TemplateEngine osProperties: $(linuxOsPortableProperties) - testProjects: $(templateEngineTestProjects) + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ### MACOS ### macOSJobParameterSets: - categoryName: TestBuild - categoryName: TemplateEngine - testProjects: $(templateEngineTestProjects) + testProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - categoryName: AoT runAoTTests: true diff --git a/eng/pipelines/templates/variables/sdk-defaults.yml b/eng/pipelines/templates/variables/sdk-defaults.yml index 803330dcb2a5..f9097cfc211a 100644 --- a/eng/pipelines/templates/variables/sdk-defaults.yml +++ b/eng/pipelines/templates/variables/sdk-defaults.yml @@ -3,7 +3,6 @@ variables: buildConfiguration: Release linuxOsPortableProperties: /p:OSName=linux /p:IsLinuxPortable=true helixTargetContainerPrefix: '@mcr.microsoft.com/dotnet-buildtools/prereqs:' - templateEngineTestProjects: $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj;$(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj ############### ARCADE ############### # "Post-build signing" means that this build will NOT sign itself. From ce7ee9cdd61d91925843c36575adca5ad7fcfac5 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 12:32:44 -0700 Subject: [PATCH 80/90] Made enableDefaultArtifacts only run on the main official windows build. --- .vsts-ci.yml | 1 + eng/pipelines/templates/jobs/sdk-build.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 185400af6ce6..686c49c93196 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -114,6 +114,7 @@ extends: publishArgument: $(_publishArgument) /p:EnableToolsetPublish=true signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) + enableDefaultArtifacts: true runTests: false publishRetryConfig: true variables: diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 8ad806f96b39..096fb7f0b753 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -20,6 +20,7 @@ parameters: signArgument: '' runTestsAsTool: false pgoInstrument: false + enableDefaultArtifacts: false runtimeIdentifier: linux-x64 osProperties: '' runtimeSourceProperties: '' @@ -77,6 +78,7 @@ jobs: -configuration $(buildConfiguration) ${{ parameters.publishArgument }} ${{ parameters.signArgument }} + /p:EnableDefaultArtifacts=${{ parameters.enableDefaultArtifacts }} /p:Architecture=${{ parameters.buildArchitecture }} /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} /p:PgoInstrument=${{ parameters.pgoInstrument }} @@ -98,7 +100,7 @@ jobs: -configuration $(buildConfiguration) \ ${{ parameters.publishArgument }} \ ${{ parameters.signArgument }} \ - /p:EnableDefaultArtifacts=false \ + /p:EnableDefaultArtifacts=${{ parameters.enableDefaultArtifacts }} \ /p:Architecture=${{ parameters.buildArchitecture }} \ /p:RunTestsAsTool=${{ parameters.runTestsAsTool }} \ /p:PgoInstrument=${{ parameters.pgoInstrument }} \ From c445c77b65ea2c74ff5148be07f6fd2ec8ca7679 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 17 May 2024 14:45:57 -0700 Subject: [PATCH 81/90] Remove no longer used templates. Remove the richnav pipeline as it has been disabled. --- .vsts-ci-richnav.yml | 86 ------- eng/build-pr.yml | 477 ------------------------------------- eng/build.yml | 514 ---------------------------------------- eng/template-engine.yml | 62 ----- 4 files changed, 1139 deletions(-) delete mode 100644 .vsts-ci-richnav.yml delete mode 100644 eng/build-pr.yml delete mode 100644 eng/build.yml delete mode 100644 eng/template-engine.yml diff --git a/.vsts-ci-richnav.yml b/.vsts-ci-richnav.yml deleted file mode 100644 index 2c07b3bb0b49..000000000000 --- a/.vsts-ci-richnav.yml +++ /dev/null @@ -1,86 +0,0 @@ -trigger: - batch: true - branches: - include: - - main - - release/*.* - -pr: none - -variables: - - name: teamName - value: Roslyn-Project-System - - name: PostBuildSign - value: true - - template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: build - displayName: Build - jobs: - - template: /eng/common/templates/job/job.yml - parameters: - name: Windows_NT_FullFramework - enableRichCodeNavigation: true - richCodeNavigationLanguage: 'csharp' - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64.open - timeoutInMinutes: 180 - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _PublishType: none - _SignType: test - _Test: -test - workspace: - clean: all - variables: - - _AgentOSName: Windows_NT_FullFramework - - _TeamName: DotNetCore - - _OfficialBuildIdArgs: '' - - _PublishArgs: '' - - _SignArgs: '' - - _InternalRuntimeDownloadArgs: '' - steps: - - powershell: eng\common\build.ps1 - -restore - -ci - -build - -pack - -sign - -configuration $(_BuildConfig) - $(_PublishArgs) - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:Test=false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - BlobFeedUrl: $(PB_PublishBlobFeedUrl) - PublishType: $(_PublishType) - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - SymStore/$(_BuildConfig)/**/* - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() diff --git a/eng/build-pr.yml b/eng/build-pr.yml deleted file mode 100644 index d3ce79f94c30..000000000000 --- a/eng/build-pr.yml +++ /dev/null @@ -1,477 +0,0 @@ -parameters: - # Agent OS identifier and used as job name - agentOs: '' - - # Agent pool - pool: {} - - # Additional variables - variables: {} - - # Build strategy - matrix - strategy: '' - - # Job timeout - timeoutInMinutes: 180 - -jobs: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0.3xx'), not(contains(parameters.agentOs, 'TemplateEngine'))) }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - CreatePr: true - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-DOTNETSDK' - MirrorBranch: release/8.0.4xx - MirrorRepo: sdk - -- ${{ if not(contains(parameters.agentOs, 'TemplateEngine')) }}: - - template: /eng/common/templates/job/job.yml - parameters: - name: ${{ parameters.agentOs }} - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ insert }}: ${{ parameters.variables }} - - _AgentOSName: ${{ parameters.agentOs }} - - _TeamName: DotNetCore - - _OfficialBuildIdArgs: '' - - _PublishArgs: '' - - _SignArgs: '' - - _HelixApiToken: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - _HelixApiToken: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - _PerfIterations: 25 - - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_PublishArgs) - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - $(_OfficialBuildArgs) - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - ${{ if eq(parameters.agentOs, 'Windows_NT_FullFramework') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - - - ${{ if eq(parameters.agentOs, 'Windows_NT_TestAsTools') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:RunTestsAsTool=true - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: - - script: eng/common/build.sh - $(_CIBuild) - --configuration $(_BuildConfig) - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/UnitTests.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() - -# AoT Jobs -- ${{ if and(in(parameters.agentOs, 'Windows_NT', 'Darwin'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}: - - template: /eng/common/templates/job/job.yml - parameters: - name: ${{ parameters.agentOs }}_AoT_Tests - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ insert }}: ${{ parameters.variables }} - - _AgentOSName: ${{ parameters.agentOs }} - - _TeamName: DotNetCore - - _OfficialBuildIdArgs: '' - - _PublishArgs: '' - - _SignArgs: '' - - _HelixApiToken: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - _HelixApiToken: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - _PerfIterations: 25 - - steps: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(parameters.agentOs, 'Windows_NT', 'Darwin')) }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update - displayName: Install wasm-tools Workload - continueOnError: false - - powershell: $(Build.SourcesDirectory)/eng/common/build.ps1 - -ci - -restore - -test - -prepareMachine - -projects $(Build.SourcesDirectory)/test/UnitTests.proj - -configuration $(_BuildConfig) - -nativeToolsOnMachine - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run AoT Tests in Helix - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'true' - - - ${{ if eq(parameters.agentOs, 'Darwin') }}: - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --restore - --build - --pack - --ci - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update - displayName: Install wasm-tools Workload - continueOnError: false - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --prepareMachine - --ci - --restore - --test - --projects $(Build.SourcesDirectory)/test/UnitTests.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run AoT Tests in Helix - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'true' - - - ${{ if in(parameters.agentOs, 'Windows_NT', 'Darwin') }}: - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() - -# TemplateEngine Jobs -- ${{ if contains(parameters.agentOs, 'TemplateEngine') }}: - - template: /eng/common/templates/job/job.yml - parameters: - name: ${{ parameters.agentOs }} - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ insert }}: ${{ parameters.variables }} - - _AgentOSName: ${{ parameters.agentOs }} - - _TeamName: DotNetCore - - _OfficialBuildIdArgs: '' - - _PublishArgs: '' - - _SignArgs: '' - - _HelixApiToken: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - _HelixApiToken: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - _PerfIterations: 25 - - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if contains(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\Microsoft.TemplateEngine.Cli.UnitTests\Microsoft.TemplateEngine.Cli.UnitTests.csproj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TemplateEngine.Cli.UnitTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run TemplateEngine Cli Unit Tests - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\dotnet-new.Tests\dotnet-new.IntegrationTests.csproj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\dotnet-new.IntegrationTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run dotnet new Integration Tests - - - ${{ if not(contains(parameters.agentOs, 'Windows_NT')) }}: - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --restore - --build - --ci - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TemplateEngine.Cli.UnitTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run TemplateEngine Cli Unit Tests - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/dotnet-new.IntegrationTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run dotnet new Integration Tests - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: xUnit - testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml' - testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' - buildPlatform: '$(BuildPlatform)' - buildConfiguration: '$(_BuildConfig)' - condition: always() - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() diff --git a/eng/build.yml b/eng/build.yml deleted file mode 100644 index d54effe04852..000000000000 --- a/eng/build.yml +++ /dev/null @@ -1,514 +0,0 @@ -parameters: - # Agent OS identifier and used as job name - agentOs: '' - # Agent pool - pool: {} - # Additional variables - variables: {} - # Build strategy - matrix - strategy: '' - # Job timeout - timeoutInMinutes: 180 - -jobs: -- ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0.3xx'), not(contains(parameters.agentOs, 'TemplateEngine'))) }}: - # Refdoc: https://github.com/dotnet/arcade/blob/main/Documentation/OneLocBuild.md#onelocbuild-template-parameters - - template: /eng/common/templates-official/job/onelocbuild.yml - parameters: - CreatePr: true - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-DOTNETSDK' - MirrorBranch: release/8.0.4xx - MirrorRepo: sdk - -- ${{ if not(contains(parameters.agentOs, 'TemplateEngine')) }}: - - template: /eng/common/templates-official/job/job.yml - parameters: - name: ${{ parameters.agentOs }} - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - name: _AgentOSName - value: ${{ parameters.agentOs }} - - name: _TeamName - value: DotNetCore - - name: _OfficialBuildIdArgs - value: '' - - name: _SignArgs - value: '' - - name: _HelixApiToken - value: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - name: _HelixApiToken - value: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - name: _OfficialBuildIdArgs - value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - name: _SignArgs - value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - name: _PerfIterations - value: 25 - - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_PublishArgs) - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - $(_OfficialBuildArgs) - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeeded() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - ${{ if eq(parameters.agentOs, 'Windows_NT_FullFramework') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)\test\UnitTests.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - TestFullMSBuild: 'true' - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - - - ${{ if eq(parameters.agentOs, 'Windows_NT_TestAsTools') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - /p:RunTestsAsTool=true - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - - ${{ if notIn(parameters.agentOs, 'Windows_NT', 'Windows_NT_FullFramework', 'Windows_NT_TestAsTools') }}: - - script: eng/common/build.sh - $(_CIBuild) - --configuration $(_BuildConfig) - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/UnitTests.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run Tests in Helix - condition: succeededOrFailed() - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'false' - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - SymStore/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() - -# AoT Jobs -- ${{ if and(in(parameters.agentOs, 'Windows_NT', 'Darwin'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}: - - template: /eng/common/templates-official/job/job.yml - parameters: - name: ${{ parameters.agentOs }}_AoT_Tests - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - name: _AgentOSName - value: ${{ parameters.agentOs }} - - name: _TeamName - value: DotNetCore - - name: _OfficialBuildIdArgs - value: '' - - name: _PublishArgs - value: '' - - name: _SignArgs - value: '' - - name: _HelixApiToken - value: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - name: _HelixApiToken - value: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - name: _OfficialBuildIdArgs - value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - name: _SignArgs - value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - name: _PerfIterations - value: 25 - - steps: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(parameters.agentOs, 'Windows_NT', 'Darwin')) }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - continueOnError: false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update - displayName: Install wasm-tools Workload - continueOnError: false - - powershell: $(Build.SourcesDirectory)/eng/common/build.ps1 - -ci - -restore - -test - -prepareMachine - -projects $(Build.SourcesDirectory)/test/UnitTests.proj - -configuration $(_BuildConfig) - -nativeToolsOnMachine - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run AoT Tests in Helix - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'true' - - - ${{ if eq(parameters.agentOs, 'Darwin') }}: - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --restore - --build - --pack - --ci - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(_BuildConfig)/dotnet/dotnet workload install wasm-tools --skip-manifest-update - displayName: Install wasm-tools Workload - continueOnError: false - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --prepareMachine - --ci - --restore - --test - --projects $(Build.SourcesDirectory)/test/UnitTests.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TestInHelix.binlog - /p:_CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} - $(_InternalRuntimeDownloadArgs) - displayName: Run AoT Tests in Helix - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: $(_HelixApiToken) - RunAoTTests: 'true' - - - ${{ if in(parameters.agentOs, 'Windows_NT', 'Darwin') }}: - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - SymStore/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() - -# TemplateEngine Jobs -- ${{ if contains(parameters.agentOs, 'TemplateEngine') }}: - - template: /eng/common/templates-official/job/job.yml - parameters: - name: ${{ parameters.agentOs }} - enableMicrobuild: true - enablePublishBuildAssets: true - enableTelemetry: true - enablePublishUsingPipelines: true - helixRepo: dotnet/sdk - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - workspace: - clean: all - variables: - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - name: _AgentOSName - value: ${{ parameters.agentOs }} - - name: _TeamName - value: DotNetCore - - name: _OfficialBuildIdArgs - value: '' - - name: _PublishArgs - value: '' - - name: _SignArgs - value: '' - - name: _HelixApiToken - value: '' - # Helix Testing requires a token when internally run - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNet-HelixApi-Access - - name: _HelixApiToken - value: $(HelixApiAccessToken) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: dotnet-benchview - - name: _OfficialBuildIdArgs - value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - name: _SignArgs - value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - - name: _PerfIterations - value: 25 - - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if contains(parameters.agentOs, 'Windows_NT') }}: - - powershell: eng\common\build.ps1 - $(_CIBuild) - -configuration $(_BuildConfig) - -nativeToolsOnMachine - $(_SignArgs) - $(_OfficialBuildIdArgs) - /p:Test=false - $(_InternalRuntimeDownloadArgs) - displayName: Build - env: - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HelixAccessToken: ${{ parameters.HelixAccessToken }} - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\Microsoft.TemplateEngine.Cli.UnitTests\Microsoft.TemplateEngine.Cli.UnitTests.csproj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\TemplateEngine.Cli.UnitTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run TemplateEngine Cli Unit Tests - - powershell: eng\common\build.ps1 - -configuration $(_BuildConfig) - -nativeToolsOnMachine - -prepareMachine - -ci - -restore - $(_Test) - -projects $(Build.SourcesDirectory)\test\dotnet-new.Tests\dotnet-new.IntegrationTests.csproj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\dotnet-new.IntegrationTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run dotnet new Integration Tests - - - ${{ if not(contains(parameters.agentOs, 'Windows_NT')) }}: - - script: eng/common/build.sh - --configuration $(_BuildConfig) - --restore - --build - --ci - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_InternalRuntimeDownloadArgs) - -p:Test=false - displayName: Build - env: - BuildConfig: $(_BuildConfig) - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/Microsoft.TemplateEngine.Cli.UnitTests/Microsoft.TemplateEngine.Cli.UnitTests.csproj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/TemplateEngine.Cli.UnitTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run TemplateEngine Cli Unit Tests - - powershell: eng/common/build.sh - -configuration $(_BuildConfig) - -prepareMachine - -ci - -restore - -test - -projects $(Build.SourcesDirectory)/test/dotnet-new.Tests/dotnet-new.IntegrationTests.csproj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/dotnet-new.IntegrationTests.binlog - $(_InternalRuntimeDownloadArgs) - displayName: Run dotnet new Integration Tests - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: xUnit - testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml' - testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' - buildPlatform: '$(BuildPlatform)' - buildConfiguration: '$(_BuildConfig)' - condition: always() - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* - SymStore/$(_BuildConfig)/**/* - tmp/$(_BuildConfig)/**/*.binlog - TargetFolder: '$(Build.ArtifactStagingDirectory)' - continueOnError: true - condition: always() - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' - publishLocation: Container - continueOnError: true - condition: always() diff --git a/eng/template-engine.yml b/eng/template-engine.yml deleted file mode 100644 index e30576650910..000000000000 --- a/eng/template-engine.yml +++ /dev/null @@ -1,62 +0,0 @@ -jobs: - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build-pr.yml - parameters: - agentOs: Windows_NT_TemplateEngine - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64.open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Windows.Amd64.VS2022.Pre - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '-publish /p:DotNetPublishUsingPipelines=true' - _SignType: test - _Test: -test - - - template: /eng/build-pr.yml - parameters: - agentOs: Ubuntu_22_04_TemplateEngine - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals build.ubuntu.2204.amd64.open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals build.ubuntu.2204.amd64 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: 'ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64' - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: Ubuntu.2204.Amd64 - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - _SignType: test - _Test: -test - - - template: /eng/build-pr.yml - parameters: - agentOs: Darwin_TemplateEngine - pool: - vmImage: 'macOS-latest' - ${{ if eq(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.1100.Amd64.Open - ${{ if ne(variables['System.TeamProject'], 'public') }}: - helixTargetQueue: OSX.1100.Amd64 - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - _SignType: test - _Test: -test \ No newline at end of file From 7d0721080bce1b3b7df54fb3bb4e34cc950e9184 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 08:34:54 +0200 Subject: [PATCH 82/90] Publish fsharp and roslyn packages only on the default artifacts leg --- eng/Publishing.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 9b70d0876e26..759f7ac26553 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -69,7 +69,7 @@ $(PublishDependsOnTargets);_ResolvePublishFSharpNuGetPackages - + Shipping Release @@ -87,7 +87,7 @@ $(PublishDependsOnTargets);_ResolvePublishRoslynNuGetPackages - + @@ -95,7 +95,7 @@ - + Date: Tue, 21 May 2024 10:09:48 +0200 Subject: [PATCH 83/90] Update Directory.Build.props --- Directory.Build.props | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 2c0593ea07fd..232846cb33f2 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -56,11 +56,6 @@ true - - true - -pgo - - false false From 0407b3f3e2d88668c2c1931e975d5d2250fed704 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 10:10:59 +0200 Subject: [PATCH 84/90] Update .vsts-ci.yml --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 686c49c93196..3e55e0e22b9f 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -111,7 +111,7 @@ extends: windowsJobParameterSets: ### OFFICIAL ### - categoryName: Official - publishArgument: $(_publishArgument) /p:EnableToolsetPublish=true + publishArgument: $(_publishArgument) signArgument: $(_signArgument) officialBuildProperties: $(_officialBuildProperties) enableDefaultArtifacts: true From f256ddd49337a1b86734fcefe9f766f728af90a8 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 10:28:40 +0200 Subject: [PATCH 85/90] Correctly set DotNetReleaseShipping=true in Publishing.props --- eng/Publishing.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 759f7ac26553..31221e5b1c8c 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -201,6 +201,7 @@ $(BlobStoragePartialRelativePath)/$(FullNugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)')) + DotNetReleaseShipping=true @@ -210,6 +211,7 @@ $(BlobStoragePartialRelativePath)/$(FullNugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)')) + DotNetReleaseShipping=true From 379cbc7f161d8b2ce4efc061438d184a1ef514b1 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 10:56:54 +0000 Subject: [PATCH 86/90] Always sign and publish .wixpack.zip --- eng/Publishing.props | 3 --- eng/Signing.props | 8 +------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 31221e5b1c8c..e5365cfb8242 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -177,11 +177,8 @@ - - - diff --git a/eng/Signing.props b/eng/Signing.props index 9c8c5453c975..d21cce9aa953 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -35,7 +35,7 @@ - + @@ -87,10 +87,4 @@ - - - - - From 700e4e5dd8312c71d3dc84888bd23d2b80e65bb7 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 11:19:28 +0000 Subject: [PATCH 87/90] Include missing VS non-windows-x64 packages --- eng/Publishing.props | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index e5365cfb8242..8610a87a8b79 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -51,6 +51,13 @@ + + + + + From 2a72654afc4531e70e03db7ea9c1d12eb955fae7 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 16:44:53 +0000 Subject: [PATCH 88/90] disable formatting sdk repo --- eng/dotnet-format/dotnet-format-integration.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/dotnet-format/dotnet-format-integration.yml b/eng/dotnet-format/dotnet-format-integration.yml index 602b2d7d2b01..af8e46e084b5 100644 --- a/eng/dotnet-format/dotnet-format-integration.yml +++ b/eng/dotnet-format/dotnet-format-integration.yml @@ -14,13 +14,14 @@ parameters: _branchName: "main" _sha: "a3bb37003aeccad012a6e7dd220977599e8b8e65" _useParentSdk: 0 - - Name: sdk - _repo: "https://github.com/dotnet/sdk" - _repoName: "dotnet/sdk" - _targetSolution: "sdk.sln" - _branchName: "main" - _sha: "be25db95c376bffd508a023399ddd34392fe6458" - _useParentSdk: 0 + # Formatting sdk fails: https://github.com/dotnet/sdk/issues/41060 + # - Name: sdk + # _repo: "https://github.com/dotnet/sdk" + # _repoName: "dotnet/sdk" + # _targetSolution: "sdk.sln" + # _branchName: "main" + # _sha: "be25db95c376bffd508a023399ddd34392fe6458" + # _useParentSdk: 0 - Name: project_system _repo: "https://github.com/dotnet/project-system" _repoName: "dotnet/project-system" From 67a92fe674eedb838e4aae900875956d17e8489d Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Tue, 21 May 2024 22:34:42 +0000 Subject: [PATCH 89/90] Fix DetermineSourceBuiltSdkVersion target --- src/SourceBuild/content/Directory.Build.targets | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets index 41405bafd5f7..79bb098deca2 100644 --- a/src/SourceBuild/content/Directory.Build.targets +++ b/src/SourceBuild/content/Directory.Build.targets @@ -8,9 +8,13 @@ - + + + +