Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
173 commits
Select commit Hold shift + click to select a range
ae66e2e
Download pipeline with auth. GH API
jpfeuffer Jun 5, 2025
191f685
remove debug
jpfeuffer Jun 5, 2025
eff74f4
ruff linter
jpfeuffer Jun 5, 2025
dfaf3fe
good old indent on empty line
jpfeuffer Jun 5, 2025
9ab9642
read topdir from zip for rename
jpfeuffer Jun 5, 2025
bb094e2
ruff....
jpfeuffer Jun 5, 2025
b423285
add a cli flag to avoid rate limit when unauthenticated
jpfeuffer Jun 6, 2025
146632f
fix test
jpfeuffer Jun 6, 2025
651a0ca
fix test: Remove api-download param from test dict since it's a boole…
jpfeuffer Sep 18, 2025
592d621
add testing artifacts to gitignore
jpfeuffer Sep 18, 2025
e77b261
lint
jpfeuffer Sep 18, 2025
8fc05a5
update module and subworkflow template
nvnieuwk Sep 26, 2025
de4908f
[automated] Update CHANGELOG.md
nf-core-bot Sep 26, 2025
4d94180
Merge branch 'dev' into feat/topics
nvnieuwk Sep 29, 2025
c48f7ac
also emit versions
nvnieuwk Sep 29, 2025
b534065
Merge branch 'dev' into feat/topics
nvnieuwk Sep 30, 2025
62ea7f0
add main.nf linting for topics
nvnieuwk Sep 30, 2025
e490c68
Update modules lint --fix to work with eval
nvnieuwk Sep 30, 2025
5f1af90
Update nf_core/module-template/main.nf
nvnieuwk Oct 1, 2025
1c6f6c9
versions emit should now contain the tool name
nvnieuwk Oct 1, 2025
a749470
Merge branch 'feat/topics' of github.com:nvnieuwk/tools into feat/topics
nvnieuwk Oct 1, 2025
6cb96ee
linting fixes for new versions topics
nvnieuwk Oct 1, 2025
a4d8aac
Merge branch 'dev' into feat/topics
nvnieuwk Oct 8, 2025
5a31432
add topic handling to the main workflow
nvnieuwk Oct 8, 2025
0e726e1
pre-commit
nvnieuwk Oct 8, 2025
3cae307
Improve file ignores in workflow file enumeration
jmuhlich Oct 8, 2025
5c6c50d
Update CHANGELOG.md
jmuhlich Oct 9, 2025
249c8eb
add versions file as valid topic output + fix meta yaml template
nvnieuwk Oct 9, 2025
6b02333
update meta yaml versions output on module create
nvnieuwk Oct 9, 2025
f9f42a1
Merge branch 'dev' into feat/topics
nvnieuwk Oct 9, 2025
3095597
pre-commit
nvnieuwk Oct 9, 2025
0eea1b4
pre-commit
nvnieuwk Oct 9, 2025
1d49215
Address PR review comments: refactor download functionality
jpfeuffer Oct 10, 2025
09a8fb0
remove duplication
jpfeuffer Oct 10, 2025
7310e75
better code flow.
jpfeuffer Oct 10, 2025
9b8d397
fmt
jpfeuffer Oct 10, 2025
2d49346
sort versions output
nvnieuwk Oct 13, 2025
c7be4d7
try to add versions on module creation
nvnieuwk Oct 13, 2025
c113b81
Merge branch 'dev' into jpfeuffer-patch-1
JulianFlesch Oct 13, 2025
5e1928d
update topics structure + added a check for empty input and output
nvnieuwk Oct 14, 2025
53f356c
Merge branch 'dev' into feat/topics
nvnieuwk Oct 14, 2025
b150244
fix wrongly resolved merge conflict
nvnieuwk Oct 14, 2025
7c954d6
fix module create tests + pre-commit
nvnieuwk Oct 14, 2025
db8e598
Adds installing nf-core at state from local workspace
JulianFlesch Oct 15, 2025
fbe8f63
fix usage of versions string
nvnieuwk Oct 16, 2025
58d4ba7
Merge branch 'dev' into feat/topics
nvnieuwk Oct 16, 2025
6fb78b8
bump version to 3.5.0dev
mashehu Oct 16, 2025
6d6c996
Merge pull request #3839 from mashehu/bump-version-back
mashehu Oct 16, 2025
89a028d
Remove building nfcore/devcontainer:dev
JulianFlesch Oct 17, 2025
f3b23e8
Update .devcontainer/setup.sh
JulianFlesch Oct 19, 2025
aae7f41
Revert "Remove building nfcore/devcontainer:dev"
JulianFlesch Oct 19, 2025
bbdcc80
Merge branch 'issues/3837-impr-devcontainer' of github.com:JulianFles…
JulianFlesch Oct 20, 2025
3f2efdc
Merge branch 'dev' into issues/3837-impr-devcontainer
mashehu Oct 24, 2025
0833d10
Merge branch 'dev' into get-wf-files-ignores
mashehu Oct 24, 2025
8107706
Add test for utils.get_wf_files
jmuhlich Oct 24, 2025
06fc1a4
Remove manually changing devcontainer tags from Checklist
JulianFlesch Oct 27, 2025
331ca17
Fix missing ) and correct ID in release notification GA
ewels Oct 27, 2025
914ca61
Merge branch 'dev' into feat/topics
nvnieuwk Oct 28, 2025
93f8cc1
re-allow versions yaml in linting
nvnieuwk Oct 28, 2025
ae22b17
don't close open template update PRs
mashehu Oct 28, 2025
75e534e
update multiqc version to fix utils test (#3853)
mashehu Oct 28, 2025
43ef567
Merge branch 'dev' into fix-mastodon-toot
mashehu Oct 28, 2025
2b6b60c
Automatically sync the Rocrate and README content during pipelines li…
ningyuxin1999 Oct 28, 2025
24d2916
Merge pull request #3850 from ewels/fix-mastodon-toot
mashehu Oct 28, 2025
8d07eec
Merge branch 'dev' into keep-sync-pr-open
ningyuxin1999 Oct 28, 2025
d04574d
Merge branch 'dev' into issues/3837-impr-devcontainer
ningyuxin1999 Oct 28, 2025
31ae184
add topic linting check
nvnieuwk Oct 28, 2025
efd1176
fix review comments
nvnieuwk Oct 29, 2025
8ffc35f
Merge branch 'dev' into feat/topics
nvnieuwk Oct 29, 2025
4500ae5
fix linting
nvnieuwk Oct 29, 2025
5e39aa4
Merge branch 'feat/topics' of github.com:nvnieuwk/tools into feat/topics
nvnieuwk Oct 29, 2025
da4cb3f
Merge pull request #3779 from nvnieuwk/feat/topics
nvnieuwk Oct 29, 2025
2518bbf
Fix changelog for topics
nvnieuwk Oct 29, 2025
2ad9c9a
[automated] Update CHANGELOG.md
nf-core-bot Oct 29, 2025
d126c02
remove automated changelog addition
nvnieuwk Oct 29, 2025
348d115
Merge pull request #3856 from nf-core/nvnieuwk-patch-1
mashehu Oct 31, 2025
de08834
add optional link to blogpost to sync PR (#3852)
mashehu Oct 31, 2025
8b60a83
Update dependency textual-dev to v1.8.0
renovate[bot] Nov 1, 2025
549e9ad
Update pre-commit hook astral-sh/ruff-pre-commit to v0.14.3
renovate[bot] Nov 1, 2025
2b52b01
[automated] Update CHANGELOG.md
nf-core-bot Nov 1, 2025
0c46390
[automated] Update CHANGELOG.md
nf-core-bot Nov 1, 2025
c47c939
Merge pull request #3861 from nf-core/renovate/astral-sh-ruff-pre-com…
mashehu Nov 3, 2025
df0c150
Merge branch 'dev' into renovate/textual-dev-1.x
mashehu Nov 3, 2025
884c849
Merge pull request #3860 from nf-core/renovate/textual-dev-1.x
mashehu Nov 3, 2025
98ee5aa
Change GitHub Codespaces badge style
maxulysse Nov 4, 2025
e897920
[automated] Update CHANGELOG.md
nf-core-bot Nov 4, 2025
3a6c87b
add note about changed PR closing behaviour
mashehu Nov 4, 2025
40cca1f
Re-set aws.client.anonymous to fix pipeline submissions (irl) on AWS …
FriederikeHanssen Nov 4, 2025
fd704b7
Merge pull request #3851 from mashehu/keep-sync-pr-open
mashehu Nov 5, 2025
8b8aa46
Merge pull request #3869 from nf-core/maxulysse-patch-4
maxulysse Nov 5, 2025
b245817
ignore nf-core components during prettier linting
mashehu Oct 31, 2025
ce886ff
[automated] Update CHANGELOG.md
nf-core-bot Oct 31, 2025
7eb53cd
Add files_unchanged section to template_features.yml
mashehu Nov 5, 2025
7410a74
[automated] Update CHANGELOG.md
nf-core-bot Nov 5, 2025
abd7d45
Update CHANGELOG.md
mashehu Nov 5, 2025
47b7050
remove trailing comas from nextflow_schema.json
mirpedrol Nov 5, 2025
d9ff872
[automated] Update CHANGELOG.md
nf-core-bot Nov 5, 2025
564be56
Make bump version snapshot test more stable
rrahn Nov 3, 2025
e171533
Make test params file more stable
rrahn Nov 3, 2025
460fc46
Make test rocrate more stable
rrahn Nov 3, 2025
159b5ed
[automated] Update CHANGELOG.md
nf-core-bot Nov 5, 2025
e1983a4
Merge branch 'dev' into fix-nextflow-schema
mashehu Nov 5, 2025
df79d3f
Merge pull request #3874 from mirpedrol/fix-nextflow-schema
mirpedrol Nov 5, 2025
db8ee04
Update GitHub Actions (major) (#3849)
renovate[bot] Nov 5, 2025
2b2c951
Update python:3.14-slim Docker digest to 4ed3310 (#3862)
renovate[bot] Nov 5, 2025
0abee45
Merge pull request #3840 from JulianFlesch/issues/3837-impr-devcontainer
JulianFlesch Nov 5, 2025
8fe0b10
Read authentication status from gh_api
JulianFlesch Nov 6, 2025
bbd9f4d
Remove Redundat downloading of workflow files in favor of single requ…
JulianFlesch Nov 6, 2025
547b84a
Merge branch 'dev' into jpfeuffer-patch-1
JulianFlesch Nov 6, 2025
f457472
linting: update json schema store URL
mashehu Nov 6, 2025
659c624
[automated] Update CHANGELOG.md
nf-core-bot Nov 6, 2025
6300095
Merge branch 'dev' into jpfeuffer-patch-1
mashehu Nov 6, 2025
7599ad5
Remove the authenticated flag. Authentication will be checked automat…
JulianFlesch Nov 6, 2025
475f83b
Remove duplicated code section
JulianFlesch Nov 6, 2025
1b9d9b5
Merge branch 'jpfeuffer-patch-1' of github.com:nf-core/tools into jpf…
JulianFlesch Nov 6, 2025
21703d8
Fix download urls for tests that run with github auth. Adds an unauth…
JulianFlesch Nov 6, 2025
59053c8
Remove removed 'authenticated' param from test call
JulianFlesch Nov 6, 2025
beef14f
Fix test with with noauth by unsetting gh_api.auth
JulianFlesch Nov 6, 2025
e54481b
Fix typo
JulianFlesch Nov 6, 2025
8ff5da6
Merge pull request #3607 from nf-core/jpfeuffer-patch-1
JulianFlesch Nov 6, 2025
f0786c4
Update docker/setup-qemu-action digest to c7c5346 (#3875)
renovate[bot] Nov 6, 2025
c9d3b76
chore(deps): update python:3.14-slim docker digest to 9813eec (#3880)
renovate[bot] Nov 7, 2025
a4be08d
Preserve the value of self.modules_repo across nested calls
muffato Nov 7, 2025
fb80c7d
[automated] Update CHANGELOG.md
nf-core-bot Nov 7, 2025
9a0ba10
sync: Avoid deleting files ignored by git during `pipelines sync` (#3…
tgelafr-pfzr Nov 7, 2025
ea2e079
chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.14.4
renovate[bot] Nov 7, 2025
cc2f59f
[automated] Update CHANGELOG.md
nf-core-bot Nov 7, 2025
9da30b8
add missing setup steps to snapshot update action
mashehu Nov 7, 2025
fbae502
remove redundant if clause
mashehu Nov 7, 2025
e535731
[automated] Update CHANGELOG.md
nf-core-bot Nov 7, 2025
7dff5ee
Update dependency textual to v6.5.0 (#3859)
renovate[bot] Nov 7, 2025
8bce834
update multiqc to 1.32
mashehu Nov 6, 2025
b81aa25
[automated] Update CHANGELOG.md
nf-core-bot Nov 6, 2025
5ab6be3
update snapshots
mashehu Nov 7, 2025
c2151f1
fix sync test
mashehu Nov 7, 2025
d096856
[automated] Update CHANGELOG.md
nf-core-bot Nov 7, 2025
e101b4f
Add Changelog entry for PR 3607
JulianFlesch Nov 6, 2025
b4b6a73
fix syntax in dockerfile for devcontainer (#3887)
mashehu Nov 10, 2025
6caa62e
Added a test
muffato Nov 11, 2025
ac1b9cd
Change information on the create pipeline landing page
jfy133 Nov 11, 2025
18f4c01
[automated] Update CHANGELOG.md
nf-core-bot Nov 11, 2025
0049c6d
one more fix forsyntax in dockerfile for devcontainer
mashehu Nov 11, 2025
772b846
Update dependency textual to v6.6.0 (#3892)
renovate[bot] Nov 11, 2025
6cd0a50
chore(deps): update mcr.microsoft.com/devcontainers/miniconda docker …
renovate[bot] Nov 11, 2025
9c306ee
chore(deps): update mcr.microsoft.com/devcontainers/base:debian docke…
renovate[bot] Nov 11, 2025
ea63385
Fix pytest matrix setup (#3888)
rrahn Nov 12, 2025
ff03f25
add word boundary for input, output and topic linting
mashehu Nov 11, 2025
dae176a
[automated] Update CHANGELOG.md
nf-core-bot Nov 11, 2025
a596d23
Update tests/modules/lint/test_main_nf.py
mashehu Nov 12, 2025
e8b2f8e
Update pre-commit hook astral-sh/ruff-pre-commit to v0.14.5 (#3900)
renovate[bot] Nov 13, 2025
e3acb62
Merge pull request #3881 from nf-core/modules_repo_nested_crossorg_su…
muffato Nov 14, 2025
d52b8b3
Enable CI tests for pipelines/download
rrahn Nov 13, 2025
b9f76e0
Fix docker test
rrahn Nov 13, 2025
734a5e7
Ensure container out dir exists
rrahn Nov 13, 2025
bae5110
Fix singularity error
rrahn Nov 13, 2025
5b78790
add linting of topics
mirpedrol Nov 14, 2025
8cf07f5
[automated] Update CHANGELOG.md
nf-core-bot Nov 14, 2025
062725c
apply new topics structure to test fixtures
mashehu Nov 17, 2025
3619d80
Fix GH API rate limits. (#3895)
rrahn Nov 17, 2025
36a5c73
Merge pull request #3902 from mirpedrol/fix-eval-parsing
mirpedrol Nov 17, 2025
01acabf
Swich to Apptainer in pytest
rrahn Nov 18, 2025
ef733fa
Update actions/checkout digest to 93cb6ef (#3906)
renovate[bot] Nov 18, 2025
530a7a1
devcontainer: Set `moby: false` to fix trixie builds (#3904)
mashehu Nov 18, 2025
55b39f6
Merge branch 'dev' into get-wf-files-ignores
mashehu Nov 18, 2025
d823888
Fix LSP warnings on pipeline template (#3905)
dialvarezs Nov 18, 2025
85f2444
Merge branch 'dev' into update-pipeline-creation-info
mashehu Nov 18, 2025
9200da2
Merge pull request #3820 from jmuhlich/get-wf-files-ignores
mirpedrol Nov 18, 2025
a4ab3e9
devcontainer: downgrade to debian 12 and revert `moby:false` (#3907)
mashehu Nov 18, 2025
b4f5d50
Merge pull request #3898 from rrahn/fix/pipelines-download-tests
MatthiasZepper Nov 18, 2025
c7c62a2
update textual snapshot
mirpedrol Nov 18, 2025
b3ad622
modules lint: handle meta.ymls without topics field (#3909)
mashehu Nov 18, 2025
65dd582
Merge pull request #3891 from jfy133/update-pipeline-creation-info
mirpedrol Nov 18, 2025
c40d998
bump to 3.5.0 (#3884)
mashehu Nov 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add main.nf linting for topics
  • Loading branch information
nvnieuwk committed Sep 30, 2025
commit 62ea7f0f35e93a21a031b96eab5a2163d37a1aec
62 changes: 41 additions & 21 deletions nf_core/modules/lint/main_nf.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ def main_nf(
* The module has a process label and it is among
the standard ones.
* If a ``meta`` map is defined as one of the modules
inputs it should be defined as one of the outputs,
inputs it should be defined as one of the emits,
and be correctly configured in the ``saveAs`` function.
* The module script section should contain definitions
of ``software`` and ``prefix``
"""

inputs: list[str] = []
outputs: list[str] = []
emits: list[str] = []
topics: list[str] = []

# Check if we have a patch file affecting the 'main.nf' file
# otherwise read the lines directly from the module
Expand Down Expand Up @@ -128,8 +129,9 @@ def main_nf(
line = joint_tuple
inputs.extend(_parse_input(module, line))
if state == "output" and not _is_empty(line):
outputs += _parse_output(module, line)
outputs = list(set(outputs)) # remove duplicate 'meta's
emits += _parse_output_emits(module, line)
emits = list(set(emits)) # remove duplicate 'meta's
topics += _parse_output_topics(module, line)
if state == "when" and not _is_empty(line):
when_lines.append(line)
if state == "script" and not _is_empty(line):
Expand All @@ -140,7 +142,7 @@ def main_nf(
exec_lines.append(line)

# Check that we have required sections
if not len(outputs):
if not len(emits):
module.failed.append(("main_nf_script_outputs", "No process 'output' block found", module.main_nf))
else:
module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf))
Expand Down Expand Up @@ -175,8 +177,8 @@ def main_nf(
if inputs:
if "meta" in inputs:
module.has_meta = True
if outputs:
if "meta" in outputs:
if emits:
if "meta" in emits:
module.passed.append(
("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)
)
Expand All @@ -186,13 +188,22 @@ def main_nf(
)

# Check that a software version is emitted
if outputs:
if "versions" in outputs:
module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf))
if topics:
if "versions" in topics:
module.passed.append(("main_nf_version_topic", "Module emits software versions as topic", module.main_nf))
else:
module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf))
module.failed.append(("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf))

return inputs, outputs
if emits:
topic_versions_amount = sum(1 for t in topics if t == "versions")
emit_versions_amount = sum(1 for e in emits if e.startswith("versions"))
if topic_versions_amount == emit_versions_amount:
module.passed.append(("main_nf_version_emit", "Module emits each software version", module.main_nf))
else:
module.failed.append(("main_nf_version_emit", "Module does not have an `emit:` and `topic:` for each software version", module.main_nf))


return inputs, emits


def check_script_section(self, lines):
Expand All @@ -202,12 +213,6 @@ def check_script_section(self, lines):
"""
script = "".join(lines)

# check that process name is used for `versions.yml`
if re.search(r"\$\{\s*task\.process\s*\}", script):
self.passed.append(("main_nf_version_script", "Process name used for versions.yml", self.main_nf))
else:
self.warned.append(("main_nf_version_script", "Process name not used for versions.yml", self.main_nf))

# check for prefix (only if module has a meta map as input)
if self.has_meta:
if re.search(r"\s*prefix\s*=\s*task.ext.prefix", script):
Expand Down Expand Up @@ -607,15 +612,30 @@ def _parse_input(self, line_raw):
return inputs


def _parse_output(self, line):
def _parse_output_emits(self, line):
output = []
if "meta" in line:
output.append("meta")
if "emit:" not in line:
emit_regex = re.search(r"^.*emit:\s*([^,\s]*)", line)
if not emit_regex:
self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf))
else:
output.append(line.split("emit:")[1].strip())
output.append(emit_regex.group(1).strip())
return output

def _parse_output_topics(self,line):
output = []
if "meta" in line:
output.append("meta")
topic_regex = re.search(r"^.*topic:\s*([^,\s]*)", line)
if topic_regex:
topic_name = topic_regex.group(1).strip()
output.append(topic_name)
if topic_name == "versions":
if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line):
self.failed.append(("wrong_version_output", 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'<tool>\'), eval("<version_command>")`', self.main_nf))
if not re.search(r'emit:\s*versions\d+', line):
self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions<version_number>`, e.g.: `versions1`, `versions2`', self.main_nf))
return output


Expand Down
Loading