From 0bc9694ceea1d7053bcc086dd61b1e6bf3fb3890 Mon Sep 17 00:00:00 2001
From: Brian Weaver <weaver.299@osu.edu>
Date: Fri, 18 Jun 2021 11:14:36 -0400
Subject: [PATCH] query_all.sh improvements
---
scripts/backup_fresh.sh | 69 +++++++++++++++++++++++++++++++++++++++++
scripts/query_all.sh | 54 ++++++++++++++++++++------------
scripts/query_site.sh | 47 ++++++++++++++++++++++++++++
3 files changed, 150 insertions(+), 20 deletions(-)
create mode 100755 scripts/backup_fresh.sh
create mode 100755 scripts/query_site.sh
diff --git a/scripts/backup_fresh.sh b/scripts/backup_fresh.sh
new file mode 100755
index 0000000000..ccc90d1a52
--- /dev/null
+++ b/scripts/backup_fresh.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+if [[ -z $1 ]]; then
+ echo; echo "Usage: $0 <site.env>";
+ echo; echo;
+ exit 1;
+fi
+
+# MacOS compatibility....
+if [[ "$OSTYPE" == *"darwin"* ]]; then
+ # echo "This is MacOS. OSTYPE[$OSTYPE]";
+ if [[ -z `which gdate` ]]; then
+ echo; echo "'gdate' command not found.";
+ echo -n "The GNU version of the 'date' command is needed on MacOS. ";
+ echo "It is available as the 'gdate' command via the 'coreutils' Homebrew package.";
+ if [[ -z `which brew` ]]; then
+ echo;
+ echo "Homebrew is NOT installed. Homebrew and the 'coreutils' package need to be installed to proceed. Stopping.";
+ echo "Visit https://brew.sh/ for more information.";
+ else
+ echo; echo "Try installing the 'coreutils' package -- brew install coreutils";
+ fi
+ echo; echo;
+ exit 1;
+ else
+ DATE_CMD='gdate';
+ fi
+elif [[ "$OSTYPE" == *"linux"* ]]; then
+ # echo "This is Linux - OSTYPE[$OSTYPE]. Assuming 'date' command is available and GNU.";
+ DATE_CMD='date';
+else
+ echo; echo "Unable to determine OS type - \$OSTYPE[$OSTYPE] contains neither 'linux' nor 'darwin'. Stopping.";
+ echo; echo;
+ exit 1;
+fi
+
+
+MAX_AGE=3600; # One hour = "fresh enough"
+
+# Returns unix timestamp with decimal
+BACKUP_DATE_UNIX=`terminus backup:info --field=date --format=string $1`;
+
+if [[ -z $BACKUP_DATE_UNIX ]]; then
+ BACKUP_DATE_UNIX=0; # lol never!
+fi
+
+BACKUP_DATE_UNIX=${BACKUP_DATE_UNIX%.*}; # Strip decimal
+BACKUP_DATE_FRIENDLY=$($DATE_CMD -d @$BACKUP_DATE_UNIX);
+# echo "Backup date: $BACKUP_DATE_FRIENDLY";
+
+NOW=`$DATE_CMD +%s -u`;
+BACKUP_AGE=`expr $NOW - $BACKUP_DATE_UNIX`;
+
+if [[ $BACKUP_AGE -gt $MAX_AGE ]]; then
+ >&2 echo "Backup is stale - $BACKUP_DATE_FRIENDLY - $BACKUP_AGE seconds ago.";
+
+ # By convention, a non-zero exit code indicates an error condition.
+ # We will return the backup age in seconds, since it is stale.
+ exit $BACKUP_AGE;
+else
+ # echo "Backup is fresh enough - $BACKUP_DATE_FRIENDLY - $BACKUP_AGE seconds ago.";
+
+ # By convention, exit code 0 means "success", so we will return 0
+ # to indicate success -- the backup is fresh
+ exit 0;
+fi
+
+
+
diff --git a/scripts/query_all.sh b/scripts/query_all.sh
index a6c6baadc9..449298654c 100755
--- a/scripts/query_all.sh
+++ b/scripts/query_all.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-# Example usage:
+# Example usage:
# echo "select distinct type, count(*) from node group by 1 order by 2 desc;" > node_counts.sql
# ./query_all.sh node_counts.sql live | tee node_counts.txt
@@ -12,24 +12,25 @@
echo; echo;
exit 1;
elif [[ ! -f $1 || ! -r $1 ]]; then
- echo; echo "The filename '$1' doesn't exist, isn't readable, or isn't an ordinary file.";
+ echo; echo "SQL file: The filename '$1' doesn't exist, isn't readable, or isn't an ordinary file.";
echo; echo;
exit 1;
fi
+
# Default to 'dev' environment if none was specified
if [[ -z $2 ]]; then
- echo; echo "No environment specified, assuming dev.."; echo;
+ >&2 echo; >&2 echo "No environment specified, assuming dev.."; >&2 echo;
ENV='dev';
else
ENV=$2;
fi
+
# We use a second "inner script" because we prefer to use GNU 'parallel' for speed if it's available, and I can't
# figure out how to run a multi-command subprocess with parallel without putting it in a second script.
INNER_SCRIPT='./query_site.sh';
-
# Check $INNER_SCRIPT exists and is executable
if [[ ! -f $INNER_SCRIPT || ! -x $INNER_SCRIPT ]]; then
echo; echo "'$INNER_SCRIPT' isn't an executable script. Stopping.";
@@ -38,34 +39,47 @@
fi
+# Check if GNU parallel
+USE_PARALLEL=0;
+PARALLEL_PATH=$(which parallel); # 'parallel' command exists
+# >&2 echo "'parallel' path: $PARALLEL_PATH" > /dev/stderr;
+
+if [[ ! -z $PARALLEL_PATH ]]; then
+ PARALLEL_VERSION=$($PARALLEL_PATH --version | head -1); # get first line of version
+ # >&2 echo "'parallel' version: $PARALLEL_VERSION" > /dev/stderr;
+ if [[ $PARALLEL_VERSION == *"GNU parallel"* ]]; then
+ # >&2 echo "GNU parallel found: $PARALLEL_PATH" > /dev/stderr;
+ USE_PARALLEL=1; # it really is GNU parallel. we like it.
+ fi
+fi
+
+
# ALL sites
-echo "Get sites...";
+>&2 echo -n "Getting site list... ";
+
SITES=$(terminus site:list --upstream=$UPSTREAM --fields=name --format=string | sort);
-echo $SITES;
+# SITES="comm-d8 comparativestudies-d8 complexity-d8 cstw-d8";
+>&2 echo $SITES; >&2 echo;
# Non-sandbox sites only
# SITES=$(terminus site:list --upstream=$UPSTREAM --fields=name --format=string --filter=plan_name!=Sandbox | sort);
-# Check for GNU parallel
-USE_PARALLEL=false;
-PARALLEL_PATH=$(which parallel); # 'parallel' command exists
-echo "'parallel' path: $PARALLEL_PATH";
-if [[ ! -z $PARALLEL_PATH ]]; then
- PARALLEL_VERSION=$($PARALLEL_PATH --version); # get version
- echo "'parallel' version: $PARALLEL_VERSION";
- if [[ $PARALLEL_VERSION == *"GNU parallel"* ]]; then
- echo "GNU parallel found: $PARALLEL_PATH";
- USE_PARALLEL=true; # it really is GNU parallel. we like it.
- fi
+# Determine whether to generate backups
+QUERY_ALL_GENERATE_BACKUPS=0;
+if [[ -z $QUERY_ALL_SKIP_BACKUPS && "$ENV" == "live" ]]; then
+ QUERY_ALL_GENERATE_BACKUPS=1;
fi
+export QUERY_ALL_GENERATE_BACKUPS=1;
-if [[ "$USE_PARALLEL" == true ]]; then
+# Run it
+echo > /dev/stderr;
+if [[ $USE_PARALLEL -gt 0 ]]; then
# With parallel
- parallel --delay 0.1 --tag "$INNER_SCRIPT {}.$ENV $1" ::: $SITES
+ parallel --delay 0.1 --tag -j 36 --line-buffer "$INNER_SCRIPT {}.$ENV $1" ::: $SITES
else
- # We're going to use the inner script even without parallel
+ # We're going to use the inner script even without parallel
# because DRY and I can't be bothered to update the logic in 2 places.
for SITE_NAME in $SITES; do
diff --git a/scripts/query_site.sh b/scripts/query_site.sh
new file mode 100755
index 0000000000..7c28a9f86e
--- /dev/null
+++ b/scripts/query_site.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+if [[ -z $1 || -z $2 ]]; then
+ echo; echo "Usage: $0 <site.env> <query_file>";
+ echo; echo;
+ exit 1;
+fi;
+
+# Split site and environment to support conditional $ENV checking
+SITE_NAME=${1%.*};
+# echo "SITE_NAME: $SITE_NAME";
+ENV=${1##*.};
+# echo "ENV: $ENV";
+
+# Wake site (only if [[ $ENV == 'live' ]] if you have no sandboxes?)
+terminus env:wake $SITE_NAME.$ENV;
+
+# BACKUP FRESHNESS SCRIPT: https://gist.github.com/weaver299/46257300e53fe50b2a0b929ab721860e
+
+# # Check if backup is "fresh" "enough"... (only if [[ $ENV == 'live' ]] maybe?)
+if [[ $QUERY_ALL_GENERATE_BACKUPS -gt 0 ]]; then
+ >&2 echo "Check backup freshness.."
+ ./backup_fresh.sh $SITE_NAME.$ENV;
+ BACKUP_FRESH=$?; # capture exit code
+
+ if [[ $BACKUP_FRESH -gt 0 ]]; then
+ # 0 means "fresh enough", else the script returns the age in seconds
+ >&2 echo "Creating a fresh backup ...";
+ # echo "terminus backup:create $SITE_NAME.$ENV --keep-for=30 --element=db";
+ terminus backup:create $SITE_NAME.$ENV --keep-for=30 --element=db;
+ fi
+fi
+
+# Get 'mysql' command from terminus
+MYSQL_CMD=$(terminus connection:info --fields=mysql_command --format=string $SITE_NAME.$ENV);
+
+# Pipe our query file into it
+QUERY_CMD="cat $2 | $MYSQL_CMD";
+
+# Suppress password on command line warning
+QUERY_CMD="$QUERY_CMD 2>&1 | grep -vi 'using a password on the command line'";
+
+# Remove field headers from query output
+QUERY_CMD="$QUERY_CMD | tail +2";
+
+# Run it
+eval $QUERY_CMD;
--
GitLab