feat: loading argo config from yaml file #165
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2025 Deutsche Telekom AG ([email protected]) | |
| # | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: Create and publish a Docker image | |
| on: | |
| workflow_run: | |
| workflows: | |
| - pypi.org Release | |
| types: | |
| - completed | |
| push: | |
| branches: | |
| - "!main" | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| REGISTRY: ghcr.io | |
| # Use the actual repository owner (handles forks) | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - linux/amd64 | |
| - linux/arm64 | |
| python_version: | |
| - '3.11' | |
| - '3.12' | |
| permissions: | |
| contents: read | |
| packages: write | |
| attestations: write | |
| id-token: write | |
| outputs: | |
| version: ${{ steps.get_version.outputs.version }} | |
| major: ${{ steps.get_version.outputs.major }} | |
| minor: ${{ steps.get_version.outputs.minor }} | |
| patch: ${{ steps.get_version.outputs.patch }} | |
| steps: | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get the version | |
| id: get_version | |
| run: | | |
| version=$(grep -Po '(?<=^version = ")[^"]*' pyproject.toml) | |
| major=$(echo $version | cut -d '.' -f 1) | |
| minor=$(echo $version | cut -d '.' -f 2) | |
| patch=$(echo $version | cut -d '.' -f 3) | |
| if [[ "${GITHUB_REF##*/}" != "main" ]]; then | |
| sha=$(echo "${GITHUB_SHA}" | cut -c1-6) | |
| version="${version}.dev${sha}" | |
| fi | |
| echo "version=$version" >> $GITHUB_ENV | |
| echo "major=$major" >> $GITHUB_ENV | |
| echo "minor=$minor" >> $GITHUB_ENV | |
| echo "patch=$patch" >> $GITHUB_ENV | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| echo "major=$major" >> $GITHUB_OUTPUT | |
| echo "minor=$minor" >> $GITHUB_OUTPUT | |
| echo "patch=$patch" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: cache-mount | |
| key: cache-mount-${{ matrix.python_version }}-${{ matrix.platform }} | |
| restore-keys: | | |
| cache-mount-${{ matrix.python_version }}- | |
| cache-mount- | |
| - name: Restore Docker cache mounts | |
| uses: reproducible-containers/[email protected] | |
| with: | |
| cache-dir: cache-mount | |
| cache-map: | | |
| { | |
| "var-cache-apt": "/var/cache/apt", | |
| "var-lib-apt": "/var/lib/apt", | |
| "uv-cache": "/tmp/.cache/uv" | |
| } | |
| skip-extraction: ${{ steps.cache.outputs.cache-hit }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }} | |
| env: | |
| DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index | |
| - name: Build and push by digest | |
| id: build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.platform }} | |
| build-args: PYTHON_VERSION=${{ matrix.python_version }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| annotations: ${{ steps.meta.outputs.annotations }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| cache-from: type=gha,scope=${{ github.workflow }}-${{ matrix.python_version }}-${{ matrix.platform }} | |
| cache-to: type=gha,mode=max,scope=${{ github.workflow }}-${{ matrix.python_version }}-${{ matrix.platform }} | |
| outputs: type=image,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} | |
| load: ${{ github.event_name == 'pull_request' }} | |
| provenance: false | |
| sbom: false | |
| - name: Export digest | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| mkdir -p ${{ runner.temp }}/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "${{ runner.temp }}/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| if: github.event_name != 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digests-${{ matrix.python_version }}-${{ env.PLATFORM_PAIR }} | |
| path: ${{ runner.temp }}/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Tag image with version | |
| if: github.event_name == 'pull_request' && matrix.platform == 'linux/amd64' | |
| run: | | |
| docker tag ${{ steps.meta.outputs.tags }} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }}:${{ env.version }} | |
| - name: Save image for testing | |
| if: github.event_name == 'pull_request' && matrix.platform == 'linux/amd64' | |
| run: | | |
| docker save ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }}:${{ env.version }} -o image.tar | |
| - name: Upload image | |
| if: github.event_name == 'pull_request' && matrix.platform == 'linux/amd64' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-image-${{ matrix.python_version }} | |
| path: image.tar | |
| retention-days: 1 | |
| merge: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| strategy: | |
| matrix: | |
| python_version: | |
| - '3.11' | |
| - '3.12' | |
| needs: | |
| - build | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| version: ${{ needs.build.outputs.version }} | |
| steps: | |
| - name: Download digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ runner.temp }}/digests | |
| pattern: digests-${{ matrix.python_version }}-* | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=${{ needs.build.outputs.version }},enable={{is_default_branch}} | |
| type=raw,value=${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }},enable={{is_default_branch}} | |
| type=raw,value=${{ needs.build.outputs.major }},enable={{is_default_branch}} | |
| type=raw,value=${{ needs.build.outputs.version }},enable=${{ github.ref != 'refs/heads/main' }} | |
| - name: Create manifest list and push | |
| working-directory: ${{ runner.temp }}/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }}:${{ steps.meta.outputs.version }} | |
| test-demo-pipeline: | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python_version: | |
| - '3.11' | |
| - '3.12' | |
| needs: | |
| - build | |
| permissions: | |
| contents: read | |
| packages: read | |
| env: | |
| REGISTRY: ghcr.io | |
| container: | |
| image: ghcr.io/${{ github.repository }}-py${{ matrix.python_version }}:${{ needs.build.outputs.version }} | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| options: --user root | |
| env: | |
| MANUALMARKDOWNSTEP__FOLDER_PATH: /usr/app/demo-data | |
| WURZEL_PIPELINE: pipelinedemo:pipeline | |
| GIT_USER: ci-test | |
| GIT_MAIL: [email protected] | |
| DVC_DATA_PATH: /usr/app/output | |
| DVC_FILE: /usr/app/dvc.yaml | |
| steps: | |
| - uses: ./.github/actions/test-pipeline | |
| test-demo-pipeline-pr: | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python_version: | |
| - '3.11' | |
| - '3.12' | |
| needs: | |
| - build | |
| permissions: | |
| contents: read | |
| packages: read | |
| pull-requests: write | |
| env: | |
| REGISTRY: ghcr.io | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/test-pipeline | |
| with: | |
| image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-py${{ matrix.python_version }}:${{ needs.build.outputs.version }} | |
| - name: Move sample output for PR comment | |
| if: matrix.python_version == '3.11' | |
| run: | | |
| # The sample output was written to tmp/sample-output.json via the mounted volume | |
| # Use sudo cp because the file is owned by root from the container | |
| if [ -f tmp/sample-output.json ]; then | |
| sudo cp tmp/sample-output.json sample-output.json | |
| sudo chmod 644 sample-output.json | |
| fi | |
| - name: Comment PR with sample output | |
| if: matrix.python_version == '3.11' && hashFiles('sample-output.json') != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const sampleOutput = fs.readFileSync('sample-output.json', 'utf8'); | |
| const output = JSON.parse(sampleOutput); | |
| // Format the output nicely | |
| const comment = `## 🎉 Pipeline Test Results | |
| The e2e pipeline test completed successfully! | |
| ### Sample Output Document | |
| <details> | |
| <summary>Click to view sample output from SimpleSplitterStep</summary> | |
| \`\`\`json | |
| ${JSON.stringify(output, null, 2)} | |
| \`\`\` | |
| </details> | |
| `; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); |