From c04699487d91c285f8afdcd7c3b857170158e1d5 Mon Sep 17 00:00:00 2001
From: Tim O'Donnell <timodonnell@gmail.com>
Date: Mon, 5 Feb 2018 19:36:02 -0500
Subject: [PATCH] Add progress bar to downloads
---
mhcflurry/downloads.yml | 62 +++++++++++++++++-----------------
mhcflurry/downloads_command.py | 34 +++++++++++++++----
2 files changed, 59 insertions(+), 37 deletions(-)
diff --git a/mhcflurry/downloads.yml b/mhcflurry/downloads.yml
index ec16abf6..010aaa17 100644
--- a/mhcflurry/downloads.yml
+++ b/mhcflurry/downloads.yml
@@ -21,85 +21,85 @@ releases:
compatibility-version: 2
downloads:
- name: models_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1.20180205.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1.20180205.tar.bz2
default: true
- name: models_class1_no_mass_spec
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1_no_mass_spec.20180205.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1_no_mass_spec.20180205.tar.bz2
default: true
- name: models_class1_experiments1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/models_class1_experiments1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/models_class1_experiments1.tar.bz2
default: false
- name: cross_validation_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/cross_validation_class1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/cross_validation_class1.tar.bz2
default: false
- name: data_iedb
- url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
default: false
- name: data_published
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_published.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_published.tar.bz2
default: false
- name: data_systemhcatlas
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_systemhcatlas.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_systemhcatlas.tar.bz2
default: false
- name: data_curated
- url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.1/data_curated.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/pre-1.1/data_curated.tar.bz2
default: true
1.0.0:
compatibility-version: 2
downloads:
- name: models_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/models_class1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/models_class1.tar.bz2
default: true
- name: models_class1_experiments1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/models_class1_experiments1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/models_class1_experiments1.tar.bz2
default: false
- name: cross_validation_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/cross_validation_class1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/pre-1.0/cross_validation_class1.tar.bz2
default: false
- name: data_iedb
- url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_iedb.tar.bz2
default: false
- name: data_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_curated
- url: https://github.com/hammerlab/mhcflurry/releases/download/pre-1.0/data_curated.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/pre-1.0/data_curated.tar.bz2
default: true
0.9.2:
compatibility-version: 2
downloads:
- name: models_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.2/models_class1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.2/models_class1.tar.bz2
default: true
- name: data_curated
- url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
default: true
- name: data_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_iedb
- url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
default: false
- name: models_class1_experiments1
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.2/models_class1_experiments1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.2/models_class1_experiments1.tar.bz2
default: false
@@ -107,55 +107,55 @@ releases:
compatibility-version: 2
downloads:
- name: models_class1
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/models_class1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/models_class1.tar.bz2
default: true
- name: data_curated
- url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_curated.tar.bz2
default: true
- name: data_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/data_kim2014.tar.bz2
default: false
- name: data_iedb
- url: https://github.com/hammerlab/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
+ url: https://github.com/openvax/mhcflurry/releases/download/0.9.1/data_iedb.tar.bz2
default: false
- name: models_class1_experiments1
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.9.1/models_class1_experiments1.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.9.1/models_class1_experiments1.tar.bz2
default: false
0.2.0:
compatibility-version: 1
downloads:
- name: models_class1_allele_specific_ensemble
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_ensemble.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_ensemble.tar.bz2
default: true
- name: models_class1_allele_specific_single
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_single.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.2.0/models_class1_allele_specific_single.tar.bz2
default: false
- name: data_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
default: true
- name: data_combined_iedb_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
default: true
0.0.8:
compatibility-version: 1
downloads:
- name: models_class1_allele_specific_single
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/models_class1_allele_specific_single.no_impute.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/models_class1_allele_specific_single.no_impute.tar.bz2
default: true
- name: data_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_kim2014.tar.bz2
default: true
- name: data_combined_iedb_kim2014
- url: http://github.com/hammerlab/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
+ url: http://github.com/openvax/mhcflurry/releases/download/0.0.8/data_combined_iedb_kim2014.tar.bz2
default: true
diff --git a/mhcflurry/downloads_command.py b/mhcflurry/downloads_command.py
index 3af2fa3e..8dfc5fb6 100644
--- a/mhcflurry/downloads_command.py
+++ b/mhcflurry/downloads_command.py
@@ -28,6 +28,7 @@ from pipes import quote
import errno
import tarfile
from tempfile import mkstemp
+from tqdm import tqdm
try:
from urllib.request import urlretrieve
except ImportError:
@@ -120,6 +121,23 @@ def yes_no(boolean):
return "YES" if boolean else "NO"
+# For progress bar on download. See https://pypi.python.org/pypi/tqdm
+class TqdmUpTo(tqdm):
+ """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
+ def update_to(self, b=1, bsize=1, tsize=None):
+ """
+ b : int, optional
+ Number of blocks transferred so far [default: 1].
+ bsize : int, optional
+ Size of each block (in tqdm units) [default: 1].
+ tsize : int, optional
+ Total size (in tqdm units). If [default: None] remains unchanged.
+ """
+ if tsize is not None:
+ self.total = tsize
+ self.update(b * bsize - self.n) # will also set self.n = b * bsize
+
+
def fetch_subcommand(args):
def qprint(msg):
if not args.quiet:
@@ -167,16 +185,18 @@ def fetch_subcommand(args):
yes_no(item in items_to_fetch),
info['metadata']["url"]))
- # TODO: Add a download progress bar?
- # See http://stackoverflow.com/questions/51212/how-to-write-a-download-progress-indicator-in-python
- # Also may want to extract into somewhere temporary and then rename to
- # making an incomplete extract if the process is killed.
+ # TODO: may want to extract into somewhere temporary and then rename to
+ # avoid making an incomplete extract if the process is killed.
for item in items_to_fetch:
metadata = downloads[item]['metadata']
(temp_fd, target_path) = mkstemp()
try:
qprint("Downloading: %s" % metadata['url'])
- urlretrieve(metadata['url'], target_path)
+ urlretrieve(
+ metadata['url'],
+ target_path,
+ reporthook=TqdmUpTo(
+ unit='B', unit_scale=True, miniters=1).update_to)
qprint("Downloaded to: %s" % quote(target_path))
tar = tarfile.open(target_path, 'r:bz2')
@@ -191,7 +211,9 @@ def fetch_subcommand(args):
"Archive has suspicious names: %s" % bad_names)
result_dir = get_path(item, test_exists=False)
os.mkdir(result_dir)
- tar.extractall(path=result_dir)
+
+ for member in tqdm(tar.getmembers(), desc='Extracting'):
+ tar.extractall(path=result_dir, members=[member])
tar.close()
qprint("Extracted %d files to: %s" % (
len(names), quote(result_dir)))
--
GitLab