Skip to content

Commit 89d8626

Browse files
scwattsscwatts
committed
Merge remote-tracking branch 'nf-core/dev' into nf-core-template-merge-3.4.1
2 parents 85e8e6b + ca25765 commit 89d8626

File tree

265 files changed

+22264
-1641
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

265 files changed

+22264
-1641
lines changed

‎.github/actions/get-shards/action.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: "Get number of shards"
2+
description: "Get the number of nf-test shards for the current CI job"
3+
inputs:
4+
max_shards:
5+
description: "Maximum number of shards allowed"
6+
required: true
7+
paths:
8+
description: "Component paths to test"
9+
required: false
10+
tags:
11+
description: "Tags to pass as argument for nf-test --tag parameter"
12+
required: false
13+
outputs:
14+
shard:
15+
description: "Array of shard numbers"
16+
value: ${{ steps.shards.outputs.shard }}
17+
total_shards:
18+
description: "Total number of shards"
19+
value: ${{ steps.shards.outputs.total_shards }}
20+
runs:
21+
using: "composite"
22+
steps:
23+
- name: Install nf-test
24+
uses: nf-core/setup-nf-test@v1
25+
with:
26+
version: ${{ env.NFT_VER }}
27+
- name: Get number of shards
28+
id: shards
29+
shell: bash
30+
run: |
31+
# Run nf-test with dynamic parameter
32+
nftest_output=$(nf-test test \
33+
--profile +docker \
34+
$(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
35+
--dry-run \
36+
--ci \
37+
--changed-since HEAD^) || {
38+
echo "nf-test command failed with exit code $?"
39+
echo "Full output: $nftest_output"
40+
exit 1
41+
}
42+
echo "nf-test dry-run output: $nftest_output"
43+
44+
# Default values for shard and total_shards
45+
shard="[]"
46+
total_shards=0
47+
48+
# Check if there are related tests
49+
if echo "$nftest_output" | grep -q 'No tests to execute'; then
50+
echo "No related tests found."
51+
else
52+
# Extract the number of related tests
53+
number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
54+
if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
55+
shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
56+
shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
57+
total_shards="$shards_to_run"
58+
else
59+
echo "Unexpected output format. Falling back to default values."
60+
fi
61+
fi
62+
63+
# Write to GitHub Actions outputs
64+
echo "shard=$shard" >> $GITHUB_OUTPUT
65+
echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
66+
67+
# Debugging output
68+
echo "Final shard array: $shard"
69+
echo "Total number of shards: $total_shards"

‎.github/actions/nf-test/action.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: "nf-test Action"
2+
description: "Runs nf-test with common setup steps"
3+
inputs:
4+
profile:
5+
description: "Profile to use"
6+
required: true
7+
shard:
8+
description: "Shard number for this CI job"
9+
required: true
10+
total_shards:
11+
description: "Total number of test shards(NOT the total number of matrix jobs)"
12+
required: true
13+
paths:
14+
description: "Test paths"
15+
required: true
16+
tags:
17+
description: "Tags to pass as argument for nf-test --tag parameter"
18+
required: false
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Setup Nextflow
23+
uses: nf-core/setup-nextflow@v2
24+
with:
25+
version: "${{ env.NXF_VERSION }}"
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
29+
with:
30+
python-version: "3.13"
31+
32+
- name: Install nf-test
33+
uses: nf-core/setup-nf-test@v1
34+
with:
35+
version: "${{ env.NFT_VER }}"
36+
install-pdiff: true
37+
38+
- name: Setup apptainer
39+
if: contains(inputs.profile, 'singularity')
40+
uses: eWaterCycle/setup-apptainer@main
41+
42+
- name: Set up Singularity
43+
if: contains(inputs.profile, 'singularity')
44+
shell: bash
45+
run: |
46+
mkdir -p $NXF_SINGULARITY_CACHEDIR
47+
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
48+
49+
- name: Conda setup
50+
if: contains(inputs.profile, 'conda')
51+
uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
52+
with:
53+
auto-update-conda: true
54+
conda-solver: libmamba
55+
conda-remove-defaults: true
56+
57+
- name: Run nf-test
58+
shell: bash
59+
env:
60+
NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
61+
run: |
62+
nf-test test \
63+
--profile=+${{ inputs.profile }} \
64+
$(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
65+
--ci \
66+
--changed-since HEAD^ \
67+
--verbose \
68+
--tap=test.tap \
69+
--shard ${{ inputs.shard }}/${{ inputs.total_shards }}
70+
71+
# Save the absolute path of the test.tap file to the output
72+
echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT
73+
74+
- name: Generate test summary
75+
if: always()
76+
shell: bash
77+
run: |
78+
# Add header if it doesn't exist (using a token file to track this)
79+
if [ ! -f ".summary_header" ]; then
80+
echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY
81+
echo "" >> $GITHUB_STEP_SUMMARY
82+
echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY
83+
echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY
84+
touch .summary_header
85+
fi
86+
87+
if [ -f test.tap ]; then
88+
while IFS= read -r line; do
89+
if [[ $line =~ ^ok ]]; then
90+
test_name="${line#ok }"
91+
# Remove the test number from the beginning
92+
test_name="${test_name#* }"
93+
echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
94+
elif [[ $line =~ ^not\ ok ]]; then
95+
test_name="${line#not ok }"
96+
# Remove the test number from the beginning
97+
test_name="${test_name#* }"
98+
echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
99+
fi
100+
done < test.tap
101+
else
102+
echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
103+
fi
104+
105+
- name: Clean up
106+
if: always()
107+
shell: bash
108+
run: |
109+
sudo rm -rf /home/ubuntu/tests/

‎.github/workflows/download_pipeline.yml‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,20 +89,12 @@ jobs:
8989
echo "Initial container image count: $image_count"
9090
echo "IMAGE_COUNT_INITIAL=$image_count" >> "$GITHUB_OUTPUT"
9191
92-
- name: Run the downloaded pipeline (stub)
92+
- name: Run the downloaded pipeline
9393
id: stub_run_pipeline
94-
continue-on-error: true
9594
env:
9695
NXF_SINGULARITY_CACHEDIR: ./singularity_container_images
9796
NXF_SINGULARITY_HOME_MOUNT: true
98-
run: nextflow run ./${{needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results
99-
- name: Run the downloaded pipeline (stub run not supported)
100-
id: run_pipeline
101-
if: ${{ steps.stub_run_pipeline.outcome == 'failure' }}
102-
env:
103-
NXF_SINGULARITY_CACHEDIR: ./singularity_container_images
104-
NXF_SINGULARITY_HOME_MOUNT: true
105-
run: nextflow run ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -profile test,singularity --outdir ./results
97+
run: nextflow run ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -stub -profile test_stub,singularity --outdir ./results
10698

10799
- name: Count the downloaded number of container images
108100
id: count_afterwards

‎.github/workflows/nf-test.yml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
name: Run nf-test
2+
on:
3+
pull_request:
4+
paths-ignore:
5+
- "docs/**"
6+
- "**/meta.yml"
7+
- "**/*.md"
8+
- "**/*.png"
9+
- "**/*.svg"
10+
release:
11+
types: [published]
12+
workflow_dispatch:
13+
14+
# Cancel if a newer run is started
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
17+
cancel-in-progress: true
18+
19+
env:
20+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
21+
NFT_TAGS: "cicd"
22+
NFT_VER: "0.9.2"
23+
NFT_WORKDIR: "~"
24+
NXF_ANSI_LOG: false
25+
NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
26+
NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
27+
28+
jobs:
29+
nf-test-changes:
30+
name: nf-test-changes
31+
runs-on: # use self-hosted runners
32+
- runs-on=${{ github.run_id }}-nf-test-changes
33+
- runner=4cpu-linux-x64
34+
outputs:
35+
shard: ${{ steps.set-shards.outputs.shard }}
36+
total_shards: ${{ steps.set-shards.outputs.total_shards }}
37+
steps:
38+
- name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
39+
run: |
40+
ls -la ./
41+
rm -rf ./* || true
42+
rm -rf ./.??* || true
43+
ls -la ./
44+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
45+
with:
46+
fetch-depth: 0
47+
48+
- name: get number of shards
49+
id: set-shards
50+
uses: ./.github/actions/get-shards
51+
env:
52+
NFT_VER: ${{ env.NFT_VER }}
53+
with:
54+
max_shards: 7
55+
tags: ${{ env.NFT_TAGS }}
56+
57+
- name: debug
58+
run: |
59+
echo ${{ steps.set-shards.outputs.shard }}
60+
echo ${{ steps.set-shards.outputs.total_shards }}
61+
62+
nf-test:
63+
name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}"
64+
needs: [nf-test-changes]
65+
if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
66+
runs-on: # use self-hosted runners
67+
- runs-on=${{ github.run_id }}-nf-test
68+
- runner=4cpu-linux-x64
69+
- disk=large
70+
strategy:
71+
fail-fast: false
72+
matrix:
73+
shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }}
74+
profile: [conda, docker, singularity]
75+
isMain:
76+
- ${{ github.base_ref == 'master' || github.base_ref == 'main' }}
77+
# Exclude conda and singularity on dev
78+
exclude:
79+
- isMain: false
80+
profile: "conda"
81+
- isMain: false
82+
profile: "singularity"
83+
NXF_VER:
84+
- "24.10.5"
85+
- "latest-everything"
86+
env:
87+
NXF_ANSI_LOG: false
88+
TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
89+
90+
steps:
91+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
92+
with:
93+
fetch-depth: 0
94+
95+
- name: Run nf-test
96+
id: run_nf_test
97+
uses: ./.github/actions/nf-test
98+
continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }}
99+
env:
100+
NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
101+
with:
102+
profile: ${{ matrix.profile }}
103+
shard: ${{ matrix.shard }}
104+
total_shards: ${{ env.TOTAL_SHARDS }}
105+
tags: ${{ env.NFT_TAGS }}
106+
107+
- name: Report test status
108+
if: ${{ always() }}
109+
run: |
110+
if [[ "${{ steps.run_nf_test.outcome }}" == "failure" ]]; then
111+
echo "::error::Test with ${{ matrix.NXF_VER }} failed"
112+
# Add to workflow summary
113+
echo "## ❌ Test failed: ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" >> $GITHUB_STEP_SUMMARY
114+
if [[ "${{ matrix.NXF_VER }}" == "latest-everything" ]]; then
115+
echo "::warning::Test with latest-everything failed but will not cause workflow failure. Please check if the error is expected or if it needs fixing."
116+
fi
117+
if [[ "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
118+
exit 1
119+
fi
120+
fi
121+
122+
confirm-pass:
123+
needs: [nf-test]
124+
if: always()
125+
runs-on: # use self-hosted runners
126+
- runs-on=${{ github.run_id }}-confirm-pass
127+
- runner=2cpu-linux-x64
128+
steps:
129+
- name: One or more tests failed (excluding latest-everything)
130+
if: ${{ contains(needs.*.result, 'failure') }}
131+
run: exit 1
132+
133+
- name: One or more tests cancelled
134+
if: ${{ contains(needs.*.result, 'cancelled') }}
135+
run: exit 1
136+
137+
- name: All tests ok
138+
if: ${{ contains(needs.*.result, 'success') }}
139+
run: exit 0
140+
141+
- name: debug-print
142+
if: always()
143+
run: |
144+
echo "::group::DEBUG: `needs` Contents"
145+
echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}"
146+
echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
147+
echo "::endgroup::"

‎.nf-core.yml‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
1+
repository_type: pipeline
2+
nf_core_version: 3.4.1
3+
template:
4+
author: Stephen Watts
5+
description: A comprehensive cancer DNA/RNA analysis and reporting pipeline
6+
force: false
7+
is_nfcore: true
8+
name: oncoanalyser
9+
org: nf-core
10+
outdir: .
11+
skip_features:
12+
- gpu
13+
- nf-test
14+
version: 2.2.0dev
115
lint:
16+
actions_ci: false
217
files_exist:
318
- lib/Utils.groovy
419
- lib/WorkflowMain.groovy
@@ -13,17 +28,3 @@ lint:
1328
- params.fastp_umi_length
1429
- params.fastp_umi_skip
1530
nf_test_content: false
16-
nf_core_version: 3.4.1
17-
repository_type: pipeline
18-
template:
19-
author: Stephen Watts
20-
description: A comprehensive cancer DNA/RNA analysis and reporting pipeline
21-
force: false
22-
is_nfcore: true
23-
name: oncoanalyser
24-
org: nf-core
25-
outdir: .
26-
skip_features:
27-
- gpu
28-
- nf-test
29-
version: 2.2.0dev

0 commit comments

Comments
 (0)