E2E (NVIDIA L40S x4) Release Branch #215
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| # yamllint disable rule:line-length | |
| name: E2E (NVIDIA L40S x4) Release Branch | |
| on: | |
| schedule: | |
| - cron: '0 11 * * *' # Runs at 11AM UTC every day | |
| workflow_dispatch: {} | |
| env: | |
| TMPDIR: /home/tmp | |
| LATEST_RELEASE_BRANCH: release-v0.26 | |
| jobs: | |
| start-large-ec2-runner: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.launch-ec2-instance-with-fallback.outputs.label }} | |
| ec2-instance-id: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-id }} | |
| ec2-instance-region: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-region }} | |
| steps: | |
| - name: Checkout "launch-ec2-runner-with-fallback" in-house CI action | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: instructlab/ci-actions | |
| # clone the "ci-actions" repo to a local directory called "ci-actions", instead of | |
| # overwriting the current WORKDIR contents | |
| path: ci-actions | |
| ref: v0.2.1 | |
| sparse-checkout: | | |
| actions/launch-ec2-runner-with-fallback | |
| - name: Launch EC2 Runner with Fallback | |
| id: launch-ec2-instance-with-fallback | |
| uses: ./ci-actions/actions/launch-ec2-runner-with-fallback | |
| env: | |
| TMPDIR: "/tmp" | |
| with: | |
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| github_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| regions_config: > | |
| [ | |
| { | |
| "region": "us-east-2", | |
| "subnets": { | |
| "us-east-2a": "${{ vars.SUBNET_US_EAST_2A }}", | |
| "us-east-2b": "${{ vars.SUBNET_US_EAST_2B }}", | |
| "us-east-2c": "${{ vars.SUBNET_US_EAST_2C }}" | |
| }, | |
| "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", | |
| "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}" | |
| }, | |
| { | |
| "region": "us-east-1", | |
| "subnets": { | |
| "us-east-1a": "${{ vars.SUBNET_US_EAST_1A }}", | |
| "us-east-1b": "${{ vars.SUBNET_US_EAST_1B }}", | |
| "us-east-1c": "${{ vars.SUBNET_US_EAST_1C }}", | |
| "us-east-1d": "${{ vars.SUBNET_US_EAST_1D }}", | |
| "us-east-1e": "${{ vars.SUBNET_US_EAST_1E }}", | |
| "us-east-1f": "${{ vars.SUBNET_US_EAST_1F }}" | |
| }, | |
| "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_1 }}", | |
| "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_1 }}" | |
| } | |
| ] | |
| try_spot_instance_first: false | |
| ec2_instance_type: g6e.12xlarge | |
| aws_resource_tags: > | |
| [ | |
| {"Key": "Name", "Value": "instructlab-ci-github-large-runner"}, | |
| {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, | |
| {"Key": "GitHubRef", "Value": "${{ github.ref }}"}, | |
| {"Key": "GitHubPR", "Value": "${{ github.event.number }}"} | |
| ] | |
| e2e-large-test: | |
| needs: | |
| - start-large-ec2-runner | |
| runs-on: ${{ needs.start-large-ec2-runner.outputs.label }} | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Install Packages | |
| run: | | |
| cat /etc/os-release | |
| mkdir -p /home/tmp | |
| sudo dnf install -y gcc gcc-c++ make git-core python3.11 python3.11-devel | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| ref: ${{ env.LATEST_RELEASE_BRANCH }} | |
| - name: Install dependent PRs if needed | |
| uses: depends-on/depends-on-action@61cb3f4a0e2c8ae4b90c9448dc57c7ba9ca24c35 # main | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install ilab | |
| run: | | |
| PYTHON="python3.11" ./scripts/install-ilab-with-cuda.sh | |
| - name: Check disk before tests | |
| run: | | |
| df -h | |
| - name: Run e2e test | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| . venv/bin/activate | |
| ./scripts/e2e-ci.sh -l -p | |
| - name: Check disk after tests | |
| if: always() | |
| run: | | |
| df -h | |
| - name: Send Discord notification for failure | |
| if: failure() | |
| uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3 | |
| with: | |
| webhook: ${{ secrets.SON_OF_JEEVES_DISCORD_WEBHOOK }} | |
| status: ${{ job.status }} | |
| title: "e2e-nvidia-l40s-x4" | |
| description: | | |
| Job in **${{ github.repository }}** running on branch `${{ env.LATEST_RELEASE_BRANCH }}` completed **with failures** ❌ | |
| Click [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) to view details. | |
| color: 0xCB2431 # Red color for failure | |
| - name: Send Discord notification for success | |
| if: success() | |
| uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3 | |
| with: | |
| webhook: ${{ secrets.SON_OF_JEEVES_DISCORD_WEBHOOK }} | |
| status: ${{ job.status }} | |
| title: "e2e-nvidia-l40s-x4" | |
| description: | | |
| Job in **${{ github.repository }}** running on branch `${{ env.LATEST_RELEASE_BRANCH }}` completed **successfully** ✅ | |
| Click [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) to view details. | |
| color: 0x28A745 # Green color for success | |
| stop-large-ec2-runner: | |
| needs: | |
| - start-large-ec2-runner | |
| - e2e-large-test | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ needs.start-large-ec2-runner.outputs.ec2-instance-region }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e # v2.3.6 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start-large-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-large-ec2-runner.outputs.ec2-instance-id }} |