Skip to content
Draft
Changes from 1 commit
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
cb07e9d
WIP: Start working on wave commands: Add command dummies.
JulianFlesch Dec 2, 2025
aa40d8b
[automated] Update CHANGELOG.md
nf-core-bot Dec 2, 2025
6b73c3c
added architecture
ningyuxin1999 Dec 2, 2025
5819387
Merge branch 'dev' into feature/3952-create-waveyml
ningyuxin1999 Dec 2, 2025
e689754
wire commands with functions
ningyuxin1999 Dec 2, 2025
c8873db
Merge branch 'feature/3952-create-waveyml' of https://github.com/Juli…
ningyuxin1999 Dec 2, 2025
8328f6f
containers obj
ningyuxin1999 Dec 2, 2025
6fba8ae
Merge branch 'dev' into feature/3952-create-waveyml
ningyuxin1999 Dec 3, 2025
78358ae
Update resolve module_dir. Change to using ValueError
JulianFlesch Dec 3, 2025
4493cad
Add TODOs
JulianFlesch Dec 3, 2025
1915c15
WIP: add container listing
JulianFlesch Dec 3, 2025
109d0b8
Move container constants to global utils.
JulianFlesch Dec 3, 2025
e723d5b
Use constants from global utils. Change list_containers to return lis…
JulianFlesch Dec 3, 2025
f44757b
add dry-run to the containers create, fixed the uitls import and modu…
ningyuxin1999 Dec 3, 2025
7c369a4
Add contextmanager that extends set_wd but changes into a tempdir
JulianFlesch Dec 4, 2025
81ddaf8
Add extracting container string from a module main.nf
JulianFlesch Dec 4, 2025
882ed67
Rename method to match field name
JulianFlesch Dec 4, 2025
aa9fa01
Print container info as richt table
JulianFlesch Dec 4, 2025
3e119a4
Merge branch 'feature/3952-create-waveyml' of github.com:JulianFlesch…
JulianFlesch Dec 4, 2025
11228d4
Move calling wave to ModuleContainers class. Finish create method and…
JulianFlesch Dec 4, 2025
c08a6dd
Fix output encoding
JulianFlesch Dec 4, 2025
c6e8360
Remove comment
JulianFlesch Dec 4, 2025
4317977
Add TODOs
JulianFlesch Dec 4, 2025
e1d7f7d
buildid and scanid saved to the dict
ningyuxin1999 Dec 5, 2025
6cd3380
Merge branch 'feature/3952-create-waveyml' of github.com:JulianFlesch…
JulianFlesch Dec 5, 2025
180287e
meta yaml
ningyuxin1999 Dec 5, 2025
d6e744f
Update calling wave command to get buildId/scanId. Only add to outpu …
JulianFlesch Dec 5, 2025
ad5463c
Simplify create method
JulianFlesch Dec 5, 2025
e0edf97
Refactor to reuse key constants.
JulianFlesch Dec 5, 2025
141c2b8
Add conda-lock information to containers dict
JulianFlesch Dec 5, 2025
972cd5e
Fix space in url
JulianFlesch Dec 5, 2025
78bd273
Implement conda lock functions. Do not fail in containers_from_meta i…
JulianFlesch Dec 5, 2025
ad640fa
Simplify class
JulianFlesch Dec 5, 2025
9b821e7
Implement updating meta - WIP: sorting
JulianFlesch Dec 5, 2025
f117fca
Fix ModuleContainers initialisation
JulianFlesch Dec 5, 2025
d0f42cc
Fix expected meta.yml path
JulianFlesch Dec 5, 2025
3ccd190
Update containers in meta.yml - WIP: sorting
JulianFlesch Dec 5, 2025
cf1dcfa
update container from meta method
JulianFlesch Dec 5, 2025
76c64f2
Review: Rename conda lock methods
JulianFlesch Dec 8, 2025
6a69fb2
Fix typo
JulianFlesch Dec 8, 2025
ff23c7d
Update nf_core/modules/containers.py
JulianFlesch Dec 8, 2025
b7e5102
Review command: Remove dry_run flag for module container creation#
JulianFlesch Dec 8, 2025
2fa3538
remove unnecessary string casting
JulianFlesch Dec 8, 2025
af97d90
Review comment: Remove extra loop for conda-lock files
JulianFlesch Dec 8, 2025
77e263e
Merge
JulianFlesch Dec 8, 2025
8f3bfb1
Make wave container requests run concurrently
JulianFlesch Dec 8, 2025
3221030
fix gathering concurrent results
JulianFlesch Dec 8, 2025
6aca629
fix building conda section
JulianFlesch Dec 8, 2025
69b6424
Remove threads option.
JulianFlesch Dec 8, 2025
8aba37c
Merge branch 'dev' into feature/3952-create-waveyml
JulianFlesch Dec 9, 2025
5a29fc7
fix variable name typo
JulianFlesch Dec 15, 2025
0088b27
Add wave inspect request to obtain https-urls for singularity images
JulianFlesch Dec 15, 2025
27f135c
Only attempt retrieving image https-url if the image build is being a…
JulianFlesch Dec 15, 2025
71bfcf5
change to reading from uri from specific layer metadata in manifest i…
JulianFlesch Dec 15, 2025
aa0b297
Remove unused constants
JulianFlesch Dec 15, 2025
c849ac1
simplify extracting uri
JulianFlesch Dec 15, 2025
95a12ef
Change to building url as suggested by ewels in https://github.com/nf…
JulianFlesch Dec 15, 2025
59f7537
Remove todo
JulianFlesch Dec 15, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
containers obj
  • Loading branch information
ningyuxin1999 committed Dec 2, 2025
commit 8328f6fdb7980156945438a10d3e722a7d025ffb
109 changes: 108 additions & 1 deletion nf_core/modules/containers.py
Original file line number Diff line number Diff line change
@@ -1 +1,108 @@
import logging
import logging
from pathlib import Path

import yaml

log = logging.getLogger(__name__)


class ModuleContainers:
"""
Helpers for building, linting and listing module containers.
"""

def __init__( # not sure how accurate this is...
self,
directory: str | Path = ".",
remote_url: str | None = None,
branch: str | None = None,
no_pull: bool = False,
hide_progress: bool | None = None,
):
self.directory = Path(directory)
self.remote_url = remote_url
self.branch = branch
self.no_pull = no_pull
self.hide_progress = hide_progress

def create(self, module: str, await_: bool = False) -> list[list[str]]:
"""
Build docker and singularity containers for linux/amd64 and linux/arm64 using wave.
"""
# module_dir = self._resolve_module_dir(module)
# env_path = self._environment_path(module_dir)

commands: list[list[str]] = []
for profile in ["docker", "singularity"]:
for platform in ["linux/amd64", "linux/arm64"]:
cmd = ["wave", "--conda-file", str(env_path), "--freeze", "--platform", platform]
# here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}]
if profile == "singularity":
cmd.append("--singularity")
if await_:
cmd.append("--await")
commands.append(cmd)
return commands

# def conda_lock(self, module: str) -> list[str]:
# """
# Build a Docker linux/arm64 container and fetch the conda lock file using wave.
# """
# module_dir = self._resolve_module_dir(module)
# env_path = self._environment_path(module_dir)
# return ["wave", "--conda-file", str(env_path), "--freeze", "--platform", "linux/arm64", "--await"]

# def lint(self, module: str) -> list[str]:
# """
# Confirm containers are defined for the module.
# """
# return self._containers_from_meta(self._resolve_module_dir(module))

# def list_containers(self, module: str) -> list[str]:
# """
# Return containers defined in the module meta.yml.
# """
# return self._containers_from_meta(self._resolve_module_dir(module))

def _resolve_module_dir(self, module: str) -> Path:
if module is None:
raise UserWarning("Please specify a module name.")

module_path = Path(module)
if module_path.parts and module_path.parts[0] == "nf-core":
module_path = Path(*module_path.parts[1:])

module_dir = self.directory / "modules" / "nf-core" / module_path
if not module_dir.exists():
raise LookupError(f"Module '{module}' not found at {module_dir}")
return module_dir

@staticmethod
def _environment_path(module_dir: Path) -> Path:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

because we lint for this, I think we should be able to reuse something there.

env_path = module_dir / "environment.yml"
if not env_path.exists():
raise FileNotFoundError(f"environment.yml not found for module at {module_dir}")
return env_path

# @staticmethod
# def _containers_from_meta(module_dir: Path) -> list[str]:
# meta_path = module_dir / "meta.yml"
# if not meta_path.exists():
# raise FileNotFoundError(f"meta.yml not found for module at {module_dir}")

# with open(meta_path) as fh:
# meta = yaml.safe_load(fh) or {}

# containers = meta.get("containers")
# if containers is None:
# raise UserWarning("No containers defined in meta.yml")
# if not isinstance(containers, list):
# raise ValueError("Expected 'containers' to be a list in meta.yml")

# cleaned = [c for c in containers if c is not None and str(c).strip()]
# if len(cleaned) != len(containers):
# raise UserWarning("Empty container entries found in meta.yml")
# if len(cleaned) == 0:
# raise UserWarning("No containers defined in meta.yml")

# return cleaned
Loading