diff --git a/downloads-generation/models_class1_pan_unselected/GENERATE.sh b/downloads-generation/models_class1_pan_unselected/GENERATE.sh index 511dcae67c147be841c60f16018872c81eae968b..889eb6a434da35f83621e31e2943d1d5e8720d3d 100755 --- a/downloads-generation/models_class1_pan_unselected/GENERATE.sh +++ b/downloads-generation/models_class1_pan_unselected/GENERATE.sh @@ -21,10 +21,12 @@ else fi # Send stdout and stderr to a logfile included with the archive. -exec > >(tee -ia "$SCRATCH_DIR/$DOWNLOAD_NAME/LOG.txt") -exec 2> >(tee -ia "$SCRATCH_DIR/$DOWNLOAD_NAME/LOG.txt" >&2) +LOG="$SCRATCH_DIR/$DOWNLOAD_NAME/LOG.$(date +%s).txt" +exec > >(tee -ia "$LOG") +exec 2> >(tee -ia "$LOG" >&2) # Log some environment info +echo "Invocation: $0 $@" date pip freeze git status @@ -79,7 +81,22 @@ do done cp $SCRIPT_ABSOLUTE_PATH . -bzip2 LOG.txt -for i in $(ls LOG-worker.*.txt) ; do bzip2 $i ; done -tar -cjf "../${DOWNLOAD_NAME}.tar.bz2" * -echo "Created archive: $SCRATCH_DIR/${DOWNLOAD_NAME}.tar.bz2" +bzip2 -f "$LOG" +for i in $(ls LOG-worker.*.txt) ; do bzip2 -f $i ; done +RESULT="$SCRATCH_DIR/${DOWNLOAD_NAME}.$(date +%Y%m%d).tar.bz2" +tar -cjf "$RESULT" * +echo "Created archive: $RESULT" + +# Split into <2GB chunks for GitHub +PARTS="${RESULT}.part." +# Check for pre-existing part files and rename them. +for i in $(ls "${PARTS}"* ) +do + DEST="${i}.OLD.$(date +%s)" + echo "WARNING: already exists: $i . Moving to $DEST" + mv $i $DEST +done +split -b 2000M "$RESULT" "$PARTS" +echo "Split into parts:" +ls -lh "${PARTS}"* +