Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
70215b1
download: Scrape DSL2 style container addresses.
ewels Jan 20, 2021
36dbda7
Fix download filenames, bugtesting
ewels Jan 20, 2021
35317cf
Code refactor into smaller functions.
ewels Jan 21, 2021
616d913
Added overall progress bar for container downloads.
ewels Jan 21, 2021
1a7795f
Download progress bar tweaks
ewels Jan 21, 2021
5192e83
Download: fix tests
ewels Jan 21, 2021
4251df7
Download: image filenames cleaned à la Nextflow
ewels Jan 21, 2021
de071fd
Tidy up some logging
ewels Jan 21, 2021
d306bf2
Download: Add --force flag
ewels Jan 21, 2021
9610ef8
Few more log statements, simplify progress bar code a little
ewels Jan 22, 2021
f1de9ce
Download: Code refactor again.
ewels Jan 25, 2021
78d5b0d
Download singularity images in parallel
ewels Jan 25, 2021
b5c6ddc
Download: Use '.partial' extensions whilst downloading.
ewels Jan 25, 2021
625db05
Download: Fix tests
ewels Jan 25, 2021
7dd36f0
Download: Make ctrl-c work with multithreading
ewels Jan 31, 2021
a9e9258
Revert wait to as_completed
ewels Jan 31, 2021
ad4bc6b
Download: New --use_singularity_cache option
ewels Jan 31, 2021
aae3388
Download: add 'singularity.cacheDir' to workflow config for image paths.
ewels Feb 1, 2021
e469b94
Fix error with singularity_pull_image()
ewels Feb 1, 2021
d0e7cbc
Make singularity image directories if they don't already exist instea…
ewels Feb 1, 2021
04362e3
- not _ in cli flags, makedirs bugfix
ewels Feb 1, 2021
4a95a5b
Tidy up download cli options a little
ewels Feb 1, 2021
5bd1766
Download: Nicer logging for exceptions in download threads
ewels Feb 1, 2021
acdf5ba
Bugfix: Creating missing directories moved upstream
ewels Feb 1, 2021
c5173d4
Download: New docs and changelog
ewels Feb 1, 2021
503d75b
Changelog - link to PR
ewels Feb 1, 2021
6e943dc
Try to simplify exception handling for threaded downloads again
ewels Feb 1, 2021
154822b
Revert the download exception handling stuff as it was working before…
ewels Feb 1, 2021
0316fa0
Better verbose debug logging
ewels Feb 1, 2021
3e719a1
Minor refactor
ewels Feb 1, 2021
e28efa4
Download: Make logging for singularity-pull as pretty as for downloads
ewels Feb 1, 2021
62a37a7
fix tests
ewels Feb 1, 2021
e4a397e
Use Popen arg name for Python 3.6
ewels Feb 1, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Download: Add --force flag
Tells tool to overwrite any existing files it finds.
  • Loading branch information
ewels committed Jan 21, 2021
commit d306bf2f4edcecfb58a4d0fee22a176539d87b3c
5 changes: 3 additions & 2 deletions nf_core/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,14 +211,15 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all
default="tar.gz",
help="Compression type",
)
def download(pipeline, release, singularity, outdir, compress):
@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files")
def download(pipeline, release, singularity, outdir, compress, force):
"""
Download a pipeline, configs and singularity container.

Collects all workflow files and shared configs from nf-core/configs.
Configures the downloaded workflow to use the relative path to the configs.
"""
dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir, compress)
dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir, compress, force)
dl.download_workflow()


Expand Down
27 changes: 19 additions & 8 deletions nf_core/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@


class DownloadProgress(Progress):
"""Custom Progress bar class, allowing us to have two progress
bars with different columns / layouts.
"""

def get_renderables(self):
for task in self.tasks:
if task.fields.get("progress_type") == "summary":
Expand Down Expand Up @@ -65,7 +69,7 @@ class DownloadWorkflow(object):
outdir (str): Path to the local download directory. Defaults to None.
"""

def __init__(self, pipeline, release=None, singularity=False, outdir=None, compress_type="tar.gz"):
def __init__(self, pipeline, release=None, singularity=False, outdir=None, compress_type="tar.gz", force=False):
self.pipeline = pipeline
self.release = release
self.singularity = singularity
Expand All @@ -74,6 +78,7 @@ def __init__(self, pipeline, release=None, singularity=False, outdir=None, compr
self.compress_type = compress_type
if self.compress_type == "none":
self.compress_type = None
self.force = force

self.wf_name = None
self.wf_sha = None
Expand All @@ -98,20 +103,26 @@ def download_workflow(self):

# Set an output filename now that we have the outdir
if self.compress_type is not None:
self.output_filename = "{}.{}".format(self.outdir, self.compress_type)
summary_log.append("Output file: '{}'".format(self.output_filename))
self.output_filename = f"{self.outdir}.{self.compress_type}"
summary_log.append(f"Output file: '{self.output_filename}'")
else:
summary_log.append("Output directory: '{}'".format(self.outdir))
summary_log.append(f"Output directory: '{self.outdir}'")

# Check that the outdir doesn't already exist
if os.path.exists(self.outdir):
log.error("Output directory '{}' already exists".format(self.outdir))
sys.exit(1)
if not self.force:
log.error(f"Output directory '{self.outdir}' already exists (use [red]--force[/] to overwrite)")
sys.exit(1)
log.warning(f"Deleting existing output directory: '{self.outdir}'")
shutil.rmtree(self.outdir)

# Check that compressed output file doesn't already exist
if self.output_filename and os.path.exists(self.output_filename):
log.error("Output file '{}' already exists".format(self.output_filename))
sys.exit(1)
if not self.force:
log.error(f"Output file '{self.output_filename}' already exists (use [red]--force[/] to overwrite)")
sys.exit(1)
log.warning(f"Deleting existing output file: '{self.output_filename}'")
os.remove(self.output_filename)

# Summary log
log.info("Saving {}\n {}".format(self.pipeline, "\n ".join(summary_log)))
Expand Down