Skip to content
Snippets Groups Projects
Commit c0469948 authored by Tim O'Donnell's avatar Tim O'Donnell
Browse files

Add progress bar to downloads

parent 8b01a3aa
No related merge requests found
Hide whitespace changes
Inline Side-by-side
mhcflurry/downloads.yml
+ 31
− 31
View file @ c0469948
......@@ -21,85 +21,85 @@ releases:
compatibility-version: 2
downloads:
- name: models_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1.20180205.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1.20180205.tar.bz2
default: true
- name: models_class1_no_mass_spec
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1_no_mass_spec.20180205.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1_no_mass_spec.20180205.tar.bz2
default: true
- name: models_class1_experiments1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1_experiments1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1_experiments1.tar.bz2
default: false
- name: cross_validation_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/cross_validation_class1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/cross_validation_class1.tar.bz2
default: false
- name: data_iedb
url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
default: false
- name: data_published
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_published.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_published.tar.bz2
default: false
- name: data_systemhcatlas
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_systemhcatlas.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_systemhcatlas.tar.bz2
default: false
- name: data_curated
url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_curated.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_curated.tar.bz2
default: true
1.0.0:
compatibility-version: 2
downloads:
- name: models_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/models_class1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/models_class1.tar.bz2
default: true
- name: models_class1_experiments1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/models_class1_experiments1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/models_class1_experiments1.tar.bz2
default: false
- name: cross_validation_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/cross_validation_class1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/cross_validation_class1.tar.bz2
default: false
- name: data_iedb
url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
default: false
- name: data_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_curated
url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_curated.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_curated.tar.bz2
default: true
0.9.2:
compatibility-version: 2
downloads:
- name: models_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.2/models_class1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.2/models_class1.tar.bz2
default: true
- name: data_curated
url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
default: true
- name: data_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_iedb
url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
default: false
- name: models_class1_experiments1
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.2/models_class1_experiments1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.2/models_class1_experiments1.tar.bz2
default: false
......@@ -107,55 +107,55 @@ releases:
compatibility-version: 2
downloads:
- name: models_class1
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/models_class1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/models_class1.tar.bz2
default: true
- name: data_curated
url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
default: true
- name: data_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_iedb
url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
default: false
- name: models_class1_experiments1
url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/models_class1_experiments1.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/models_class1_experiments1.tar.bz2
default: false
0.2.0:
compatibility-version: 1
downloads:
- name: models_class1_allele_specific_ensemble
url: http://github.com/hammerlab/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_ensemble.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_ensemble.tar.bz2
default: true
- name: models_class1_allele_specific_single
url: http://github.com/hammerlab/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_single.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_single.tar.bz2
default: false
- name: data_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
default: true
- name: data_combined_iedb_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
default: true
0.0.8:
compatibility-version: 1
downloads:
- name: models_class1_allele_specific_single
url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/models_class1_allele_specific_single.no_impute.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/models_class1_allele_specific_single.no_impute.tar.bz2
default: true
- name: data_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
default: true
- name: data_combined_iedb_kim2014
url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
default: true
mhcflurry/downloads_command.py
+ 28
− 6
View file @ c0469948
......@@ -28,6 +28,7 @@ from pipes import quote
import errno
import tarfile
from tempfile import mkstemp
from tqdm import tqdm
try:
from urllib.request import urlretrieve
except ImportError:
......@@ -120,6 +121,23 @@ def yes_no(boolean):
return "YES" if boolean else "NO"
# For progress bar on download. See https://pypi.python.org/pypi/tqdm
class TqdmUpTo(tqdm):
"""Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def fetch_subcommand(args):
def qprint(msg):
if not args.quiet:
......@@ -167,16 +185,18 @@ def fetch_subcommand(args):
yes_no(item in items_to_fetch),
info['metadata']["url"]))
# TODO: Add a download progress bar?
# See http://stackoverflow.com/questions/51212/how-to-write-a-download-progress-indicator-in-python
# Also may want to extract into somewhere temporary and then rename to
# making an incomplete extract if the process is killed.
# TODO: may want to extract into somewhere temporary and then rename to
# avoid making an incomplete extract if the process is killed.
for item in items_to_fetch:
metadata = downloads[item]['metadata']
(temp_fd, target_path) = mkstemp()
try:
qprint("Downloading: %s" % metadata['url'])
urlretrieve(metadata['url'], target_path)
urlretrieve(
metadata['url'],
target_path,
reporthook=TqdmUpTo(
unit='B', unit_scale=True, miniters=1).update_to)
qprint("Downloaded to: %s" % quote(target_path))
tar = tarfile.open(target_path, 'r:bz2')
......@@ -191,7 +211,9 @@ def fetch_subcommand(args):
"Archive has suspicious names: %s" % bad_names)
result_dir = get_path(item, test_exists=False)
os.mkdir(result_dir)
tar.extractall(path=result_dir)
for member in tqdm(tar.getmembers(), desc='Extracting'):
tar.extractall(path=result_dir, members=[member])
tar.close()
qprint("Extracted %d files to: %s" % (
len(names), quote(result_dir)))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment