query_all.sh improvements

scripts/backup_fresh.sh

+#!/usr/bin/env bash
+
+if [[ -z $1 ]]; then
+  echo; echo "Usage: $0 <site.env>";
+  echo; echo;
+  exit 1;
+fi
+
+# MacOS compatibility....
+if [[ "$OSTYPE" == *"darwin"* ]]; then
+  # echo "This is MacOS.  OSTYPE[$OSTYPE]";
+  if [[ -z `which gdate` ]]; then
+    echo; echo "'gdate' command not found.";
+    echo -n "The GNU version of the 'date' command is needed on MacOS. ";
+    echo "It is available as the 'gdate' command via the 'coreutils' Homebrew package.";
+    if [[ -z `which brew` ]]; then
+      echo;
+      echo "Homebrew is NOT installed. Homebrew and the 'coreutils' package need to be installed to proceed. Stopping.";
+      echo "Visit https://brew.sh/ for more information.";
+    else
+      echo; echo "Try installing the 'coreutils' package -- brew install coreutils";
+    fi
+    echo; echo;
+    exit 1;
+  else
+    DATE_CMD='gdate';
+  fi
+elif [[ "$OSTYPE" == *"linux"* ]]; then
+  # echo "This is Linux - OSTYPE[$OSTYPE]. Assuming 'date' command is available and GNU.";
+  DATE_CMD='date';
+else
+  echo; echo "Unable to determine OS type - \$OSTYPE[$OSTYPE] contains neither 'linux' nor 'darwin'. Stopping.";
+  echo; echo;
+  exit 1;
+fi
+
+
+MAX_AGE=3600;  # One hour = "fresh enough"
+
+# Returns unix timestamp with decimal
+BACKUP_DATE_UNIX=`terminus backup:info --field=date --format=string $1`;
+
+if [[ -z $BACKUP_DATE_UNIX ]]; then
+  BACKUP_DATE_UNIX=0;   # lol never!
+fi
+
+BACKUP_DATE_UNIX=${BACKUP_DATE_UNIX%.*};  # Strip decimal
+BACKUP_DATE_FRIENDLY=$($DATE_CMD -d @$BACKUP_DATE_UNIX);
+# echo "Backup date: $BACKUP_DATE_FRIENDLY";
+
+NOW=`$DATE_CMD +%s -u`;
+BACKUP_AGE=`expr $NOW - $BACKUP_DATE_UNIX`;
+
+if [[ $BACKUP_AGE -gt $MAX_AGE ]]; then
+  >&2 echo "Backup is stale - $BACKUP_DATE_FRIENDLY - $BACKUP_AGE seconds ago.";
+
+  # By convention, a non-zero exit code indicates an error condition.
+  # We will return the backup age in seconds, since it is stale.
+  exit $BACKUP_AGE;
+else
+  # echo "Backup is fresh enough - $BACKUP_DATE_FRIENDLY - $BACKUP_AGE seconds ago.";
+
+  # By convention, exit code 0 means "success", so we will return 0
+  # to indicate success -- the backup is fresh
+  exit 0;
+fi
+
+
+

scripts/query_all.sh

 #!/usr/bin/env bash
 
-# Example usage: 
+# Example usage:
 #   echo "select distinct type, count(*) from node group by 1 order by 2 desc;" > node_counts.sql
 #   ./query_all.sh node_counts.sql live | tee node_counts.txt
 
@@ -12,24 +12,25 @@
   echo; echo;
   exit 1;
 elif [[ ! -f $1 || ! -r $1 ]]; then
-  echo; echo "The filename '$1' doesn't exist, isn't readable, or isn't an ordinary file.";
+  echo; echo "SQL file: The filename '$1' doesn't exist, isn't readable, or isn't an ordinary file.";
   echo; echo;
   exit 1;
 fi
 
+
 # Default to 'dev' environment if none was specified
 if [[ -z $2 ]]; then
-  echo; echo "No environment specified, assuming dev.."; echo;
+  >&2 echo; >&2 echo "No environment specified, assuming dev.."; >&2 echo;
   ENV='dev';
 else
   ENV=$2;
 fi
 
+
 # We use a second "inner script" because we prefer to use GNU 'parallel' for speed if it's available, and I can't
 # figure out how to run a multi-command subprocess with parallel without putting it in a second script.
 INNER_SCRIPT='./query_site.sh';
 
-
 # Check $INNER_SCRIPT exists and is executable
 if [[ ! -f $INNER_SCRIPT || ! -x $INNER_SCRIPT ]]; then
   echo; echo "'$INNER_SCRIPT' isn't an executable script. Stopping.";
@@ -38,34 +39,47 @@
 fi
 
 
+# Check if GNU parallel
+USE_PARALLEL=0;
+PARALLEL_PATH=$(which parallel);   # 'parallel' command exists
+# >&2 echo "'parallel' path: $PARALLEL_PATH" > /dev/stderr;
+
+if [[ ! -z $PARALLEL_PATH ]]; then
+  PARALLEL_VERSION=$($PARALLEL_PATH --version | head -1); # get first line of version
+  # >&2 echo "'parallel' version: $PARALLEL_VERSION" > /dev/stderr;
+  if [[ $PARALLEL_VERSION == *"GNU parallel"* ]]; then
+    # >&2 echo "GNU parallel found: $PARALLEL_PATH" > /dev/stderr;
+    USE_PARALLEL=1;   # it really is GNU parallel.  we like it.
+  fi
+fi
+
+
 # ALL sites
-echo "Get sites...";
+>&2 echo -n "Getting site list... ";
+
 SITES=$(terminus site:list --upstream=$UPSTREAM --fields=name --format=string | sort);
-echo $SITES;
+# SITES="comm-d8 comparativestudies-d8 complexity-d8 cstw-d8";
+>&2 echo $SITES; >&2 echo;
 
 # Non-sandbox sites only
 # SITES=$(terminus site:list --upstream=$UPSTREAM --fields=name --format=string --filter=plan_name!=Sandbox | sort);
 
-# Check for GNU parallel
-USE_PARALLEL=false;
-PARALLEL_PATH=$(which parallel);   # 'parallel' command exists
-echo "'parallel' path: $PARALLEL_PATH";
 
-if [[ ! -z $PARALLEL_PATH ]]; then
-  PARALLEL_VERSION=$($PARALLEL_PATH --version);   # get version
-  echo "'parallel' version: $PARALLEL_VERSION";
-  if [[ $PARALLEL_VERSION == *"GNU parallel"* ]]; then
-    echo "GNU parallel found: $PARALLEL_PATH";
-    USE_PARALLEL=true;   # it really is GNU parallel.  we like it.
-  fi
+# Determine whether to generate backups
+QUERY_ALL_GENERATE_BACKUPS=0;
+if [[ -z $QUERY_ALL_SKIP_BACKUPS && "$ENV" == "live" ]]; then
+  QUERY_ALL_GENERATE_BACKUPS=1;
 fi
+export QUERY_ALL_GENERATE_BACKUPS=1;
 
 
-if [[ "$USE_PARALLEL" == true ]]; then
+# Run it
+echo > /dev/stderr;
+if [[ $USE_PARALLEL -gt 0 ]]; then
   # With parallel
-  parallel --delay 0.1 --tag "$INNER_SCRIPT {}.$ENV $1" ::: $SITES
+  parallel --delay 0.1 --tag -j 36 --line-buffer "$INNER_SCRIPT {}.$ENV $1" ::: $SITES
 else
-  # We're going to use the inner script even without parallel 
+  # We're going to use the inner script even without parallel
   # because DRY and I can't be bothered to update the logic in 2 places.
 
   for SITE_NAME in $SITES; do

scripts/query_site.sh

+#!/usr/bin/env bash
+
+if [[ -z $1 || -z $2 ]]; then
+    echo; echo "Usage: $0 <site.env> <query_file>";
+    echo; echo;
+    exit 1;
+fi;
+
+# Split site and environment to support conditional $ENV checking
+SITE_NAME=${1%.*};
+# echo "SITE_NAME: $SITE_NAME";
+ENV=${1##*.};
+# echo "ENV: $ENV";
+
+# Wake site  (only if [[ $ENV == 'live' ]] if you have no sandboxes?)
+terminus env:wake $SITE_NAME.$ENV;
+
+# BACKUP FRESHNESS SCRIPT: https://gist.github.com/weaver299/46257300e53fe50b2a0b929ab721860e
+
+# # Check if backup is "fresh" "enough"... (only if [[ $ENV == 'live' ]] maybe?)
+if [[ $QUERY_ALL_GENERATE_BACKUPS -gt 0 ]]; then
+  >&2 echo "Check backup freshness.."
+  ./backup_fresh.sh $SITE_NAME.$ENV;
+  BACKUP_FRESH=$?; # capture exit code
+
+  if [[ $BACKUP_FRESH -gt 0 ]]; then
+    # 0 means "fresh enough", else the script returns the age in seconds
+    >&2 echo "Creating a fresh backup ...";
+    # echo "terminus backup:create $SITE_NAME.$ENV --keep-for=30 --element=db";
+    terminus backup:create $SITE_NAME.$ENV --keep-for=30 --element=db;
+  fi
+fi
+
+# Get 'mysql' command from terminus
+MYSQL_CMD=$(terminus connection:info --fields=mysql_command --format=string $SITE_NAME.$ENV);
+
+# Pipe our query file into it
+QUERY_CMD="cat $2 | $MYSQL_CMD";
+
+# Suppress password on command line warning
+QUERY_CMD="$QUERY_CMD 2>&1 | grep -vi 'using a password on the command line'";
+
+# Remove field headers from query output
+QUERY_CMD="$QUERY_CMD | tail +2";
+
+# Run it
+eval $QUERY_CMD;