Nested cross-organisational subworkflows cause tools failure and clashing namespaces #3876
Description
Description of the bug
In the name of achieving ~Perfect Modularity~, I have been developing cross-organisational subworkflows using the new functionality in tools. Our repository is available here: https://github.com/sanger-tol/nf-core-modules
I have found a bug that causes an error when these subworkflows are nested, and when different levels of nesting call the same subworkflow. In this case, I have two subworkflows, which both call the same nested subworkflow (all are within our modules repo, not nf-core):
fasta_purge_retained_haplotype -> fastx_map_long_reads -> bam_samtools_merge_markdup
and
cram_map_illumina_hic -> bam_samtools_merge_markdup
You can install either of these two top-level subworflows, and things work fine. But installing both leads to a failure in nf-core tools - it seems to start to think that bam_samtools_merge_markdup also comes from the nf-core repository. Deleting the section of the modules.json file (attached) with the erroneous subworkflow in the nf-core section silences the warning, but the Python error remains.
This is reproducable in an empty test pipeline.
Command used and terminal output
nf-core pipelines create --name test --organisation sanger-tol --author meeeee --description "this is a test"
nf-core subworkflows --git-remote https://github.com/sanger-tol/nf-core-modules.git install fasta_purge_retained_haplotype
nf-core subworkflows --git-remote https://github.com/sanger-tol/nf-core-modules.git install cram_map_illumina_hic
nf-core pipelines lint nf-core/tools version 3.4.1 - https://nf-co.re
INFO Reinstalling subworkflows found in 'modules.json' but missing from directory:
'subworkflows/nf-core/bam_samtools_merge_markdup'
WARNING Could not install module 'modules/nf-core/bam_samtools_merge_markdup' - removing from modules.json
INFO Was unable to reinstall 'nf-core/bam_samtools_merge_markdup'. Removing 'modules.json' entry
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/bin/nf-core:10 in <module> │
│ │
│ 7 │
│ 8 if __name__ == '__main__': │
│ 9 │ sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) │
│ ❱ 10 │ sys.exit(run_nf_core()) │
│ 11 │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /__main__.py:134 in run_nf_core │
│ │
│ 131 │ │ │ log.debug(f"Could not check latest version: {e}") │
│ 132 │ │ stderr.print("\n") │
│ 133 │ # Launch the click cli │
│ ❱ 134 │ nf_core_cli(auto_envvar_prefix="NFCORE") │
│ 135 │
│ 136 │
│ 137 @tui(command="interface", help="Launch the nf-core interface") │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/rich_cl │
│ ick/rich_command.py:402 in __call__ │
│ │
│ 399 │ │ # Include this here because I run into a false warning │
│ 400 │ │ # in the PyCharm IDE otherwise; for some reason PyCharm doesn't │
│ 401 │ │ # seem to think RichGroups are callable. (No issues with Mypy, though.) │
│ ❱ 402 │ │ return super().__call__(*args, **kwargs) │
│ 403 │ │
│ 404 │ @overload │
│ 405 │ def command(self, __func: Callable[..., Any]) -> RichCommand: ... │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/c │
│ ore.py:1462 in __call__ │
│ │
│ 1459 │ │
│ 1460 │ def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: │
│ 1461 │ │ """Alias for :meth:`main`.""" │
│ ❱ 1462 │ │ return self.main(*args, **kwargs) │
│ 1463 │
│ 1464 │
│ 1465 class _FakeSubclassCheck(type): │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/rich_cl │
│ ick/rich_command.py:216 in main │
│ │
│ 213 │ │ try: │
│ 214 │ │ │ try: │
│ 215 │ │ │ │ with self.make_context(prog_name, args, **extra) as ctx: │
│ ❱ 216 │ │ │ │ │ rv = self.invoke(ctx) │
│ 217 │ │ │ │ │ if not standalone_mode: │
│ 218 │ │ │ │ │ │ return rv │
│ 219 │ │ │ │ │ # it's not safe to `ctx.exit(rv)` here! │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/c │
│ ore.py:1850 in invoke │
│ │
│ 1847 │ │ │ │ super().invoke(ctx) │
│ 1848 │ │ │ │ sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) │
│ 1849 │ │ │ │ with sub_ctx: │
│ ❱ 1850 │ │ │ │ │ return _process_result(sub_ctx.command.invoke(sub_ctx)) │
│ 1851 │ │ │
│ 1852 │ │ # In chain mode we create the contexts step by step, but after the │
│ 1853 │ │ # base command has been invoked. Because at that point we do not │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/c │
│ ore.py:1850 in invoke │
│ │
│ 1847 │ │ │ │ super().invoke(ctx) │
│ 1848 │ │ │ │ sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) │
│ 1849 │ │ │ │ with sub_ctx: │
│ ❱ 1850 │ │ │ │ │ return _process_result(sub_ctx.command.invoke(sub_ctx)) │
│ 1851 │ │ │
│ 1852 │ │ # In chain mode we create the contexts step by step, but after the │
│ 1853 │ │ # base command has been invoked. Because at that point we do not │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/c │
│ ore.py:1246 in invoke │
│ │
│ 1243 │ │ │ echo(style(message, fg="red"), err=True) │
│ 1244 │ │ │
│ 1245 │ │ if self.callback is not None: │
│ ❱ 1246 │ │ │ return ctx.invoke(self.callback, **ctx.params) │
│ 1247 │ │
│ 1248 │ def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: │
│ 1249 │ │ """Return a list of completions for the incomplete value. Looks │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/c │
│ ore.py:814 in invoke │
│ │
│ 811 │ │ │
│ 812 │ │ with augment_usage_errors(self): │
│ 813 │ │ │ with ctx: │
│ ❱ 814 │ │ │ │ return callback(*args, **kwargs) │
│ 815 │ │
│ 816 │ def forward(self, cmd: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: │
│ 817 │ │ """Similar to :meth:`invoke` but fills in default keyword │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/click/d │
│ ecorators.py:34 in new_func │
│ │
│ 31 │ """ │
│ 32 │ │
│ 33 │ def new_func(*args: P.args, **kwargs: P.kwargs) -> R: │
│ ❱ 34 │ │ return f(get_current_context(), *args, **kwargs) │
│ 35 │ │
│ 36 │ return update_wrapper(new_func, f) │
│ 37 │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /__main__.py:310 in command_pipelines_lint │
│ │
│ 307 │ """ │
│ 308 │ Check pipeline code against nf-core guidelines. │
│ 309 │ """ │
│ ❱ 310 │ pipelines_lint(ctx, directory, release, fix, key, show_passed, fail_ignored, fail_wa │
│ 311 │
│ 312 │
│ 313 # nf-core pipelines download │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /commands_pipelines.py:133 in pipelines_lint │
│ │
│ 130 │ │
│ 131 │ # Run the lint tests! │
│ 132 │ try: │
│ ❱ 133 │ │ lint_obj, module_lint_obj, subworkflow_lint_obj = run_linting( │
│ 134 │ │ │ directory, │
│ 135 │ │ │ release, │
│ 136 │ │ │ fix, │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /pipelines/lint/__init__.py:613 in run_linting │
│ │
│ 610 │ │ subworkflow_lint_obj = None │
│ 611 │ else: │
│ 612 │ │ # Create the modules lint object │
│ ❱ 613 │ │ module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hi │
│ 614 │ │ # Create the subworkflows lint object │
│ 615 │ │ try: │
│ 616 │ │ │ subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_di │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /modules/lint/__init__.py:70 in __init__ │
│ │
│ 67 │ │ registry: str | None = None, │
│ 68 │ │ hide_progress: bool = False, │
│ 69 │ ): │
│ ❱ 70 │ │ super().__init__( │
│ 71 │ │ │ component_type="modules", │
│ 72 │ │ │ directory=directory, │
│ 73 │ │ │ fail_warned=fail_warned, │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /components/lint/__init__.py:99 in __init__ │
│ │
│ 96 │ │ │
│ 97 │ │ if self.repo_type == "pipeline": │
│ 98 │ │ │ modules_json = ModulesJson(self.directory) │
│ ❱ 99 │ │ │ modules_json.check_up_to_date() │
│ 100 │ │ │ self.all_remote_components: list[NFCoreComponent] = [] │
│ 101 │ │ │ for repo_url, components in modules_json.get_all_components(self.component_t │
│ 102 │ │ │ │ if remote_url is not None and remote_url != repo_url: │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /modules/modules_json.py:679 in check_up_to_date │
│ │
│ 676 │ │ │ dump_modules_json = True │
│ 677 │ │ │ for repo, subworkflows in subworkflows_dict.items(): │
│ 678 │ │ │ │ for org, subworkflow in subworkflows: │
│ ❱ 679 │ │ │ │ │ self.recreate_dependencies(repo, org, {"name": subworkflow}) │
│ 680 │ │ self.pipeline_components = original_pipeline_components │
│ 681 │ │ │
│ 682 │ │ if dump_modules_json: │
│ │
│ /Users/jd42/.homebrew/Caskroom/miniforge/base/envs/nextflow/lib/python3.14/site-packages/nf_core │
│ /modules/modules_json.py:1284 in recreate_dependencies │
│ │
│ 1281 │ │ │ if installed_by == ["subworkflows"]: │
│ 1282 │ │ │ │ self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["instal │
│ 1283 │ │ │ if sw_name not in installed_by: │
│ ❱ 1284 │ │ │ │ self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["instal │
│ 1285 │ │ │ self.recreate_dependencies(repo, org, dep_subwf) │
│ 1286 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
TypeError: cannot use 'dict' as a dict key (unhashable type: 'dict')
System information
Nextflow version: 25.04
Hardware: MacOS + Linux
Tools version: 3.4.1
Python version: 3.14