Skip to content
Snippets Groups Projects
Commit fa146d87 authored by Matthew Hu's avatar Matthew Hu
Browse files

Added past 2 months to data directory 

Modified start_searches for the past month (30 days) rather than snap to last month
generate_reports.sh puts all output into data/year_month
parent 9c51aea7
No related branches found
No related tags found
1 merge request!2Refactored to use splunk
Hide whitespace changes
Inline Side-by-side
.gitignore
+ 0
− 1
View file @ fa146d87
...@@ -54,4 +54,3 @@ google_api_key.json ...@@ -54,4 +54,3 @@ google_api_key.json
# Ignore metrics data # Ignore metrics data
/output/ /output/
/data/
\ No newline at end of file
bin/metrics
+ 2
− 2
View file @ fa146d87
...@@ -289,7 +289,7 @@ class App_Data < Data_Methods ...@@ -289,7 +289,7 @@ class App_Data < Data_Methods
ranking = 1 ranking = 1
table_data = [] table_data = []
table_data << ["Rank", "User", "Name", "# Sessions\\*", "# App Launches", "# Launches / Session", "# Apps Used"] table_data << ["Rank", "User", "Name", "# Sessions\\*", "# App Launches", "# Launches / Session", "# Apps Used"]
table_data.last.insert(2, "Institution") if projects table_data.last.insert(3, "Institution") if projects #insert at 3, ranking will shift all values
top_user.each do |v| top_user.each do |v|
passwd = Etc.getpwnam(v.first) rescue nil passwd = Etc.getpwnam(v.first) rescue nil
full_name = passwd ? passwd.gecos : "Unknown" full_name = passwd ? passwd.gecos : "Unknown"
...@@ -606,4 +606,4 @@ puts "\\* Session = All activity within the #{SESSION_TIME} hour period of time ...@@ -606,4 +606,4 @@ puts "\\* Session = All activity within the #{SESSION_TIME} hour period of time
end end
main main
\ No newline at end of file
bin/start_searches
+ 2
− 2
View file @ fa146d87
...@@ -12,7 +12,7 @@ PASS = ENV["PASS"] || "pass" ...@@ -12,7 +12,7 @@ PASS = ENV["PASS"] || "pass"
SID_DIR = ENV["SID_DIR"] || "sid" SID_DIR = ENV["SID_DIR"] || "sid"
BASEURL = 'https://splunk.osc.edu:8089' BASEURL = 'https://splunk.osc.edu:8089'
BASIC_QUERY = "search source=/var/log/httpd24/#{HOST}_access_ssl.log earliest=-1w@w latest=@w | where user!=\"-\" | " BASIC_QUERY = "search source=/var/log/httpd24/#{HOST}_access_ssl.log earliest=-1mon latest=now | where user!=\"-\" | "
APP_QUERY = "#{BASIC_QUERY} where http_referer!=\"-\" | search *#{HOST}/pun/ | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+http_referer| dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\")| table mytime, user, http_referer" APP_QUERY = "#{BASIC_QUERY} where http_referer!=\"-\" | search *#{HOST}/pun/ | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+http_referer| dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\")| table mytime, user, http_referer"
CLIENT_QUERY = "#{BASIC_QUERY} lookup user_agents http_user_agent | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+ua_family | dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, ua_family, ua_major, ua_os_family, ua_os_major, ua_minor, ua_patch, ua_os_minor, ua_os_patch" CLIENT_QUERY = "#{BASIC_QUERY} lookup user_agents http_user_agent | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+ua_family | dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, ua_family, ua_major, ua_os_family, ua_os_major, ua_minor, ua_patch, ua_os_minor, ua_os_patch"
LOCATION_QUERY = "#{BASIC_QUERY} iplocation src_ip | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+City| dedup session |eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, City, Region, Country" LOCATION_QUERY = "#{BASIC_QUERY} iplocation src_ip | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+City| dedup session |eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, City, Region, Country"
...@@ -56,4 +56,4 @@ def main() ...@@ -56,4 +56,4 @@ def main()
abort "Wrong User or Password" unless var.to_i == 201 abort "Wrong User or Password" unless var.to_i == 201
end end
main main
\ No newline at end of file
generate_reports.sh
+ 22
− 24
View file @ fa146d87
...@@ -17,40 +17,28 @@ function generate_monthly_report(){ ...@@ -17,40 +17,28 @@ function generate_monthly_report(){
export END_DATE=$(date +%Y-%m-%d) export END_DATE=$(date +%Y-%m-%d)
SHORT_DATE=$(date +%Y-%m) SHORT_DATE=$(date +%Y-%m)
export APP_DATA_FILE=$OUTPUT_DIR/$APP/data.app.txt export APP_DATA_FILE=$THIS_DIR/$APP/data.app.txt
export LOCATION_DATA_FILE=$OUTPUT_DIR/$APP/data.location.txt export LOCATION_DATA_FILE=$THIS_DIR/$APP/data.location.txt
export CLIENT_DATA_FILE=$OUTPUT_DIR/$APP/data.client.txt export CLIENT_DATA_FILE=$THIS_DIR/$APP/data.client.txt
export PROJ_DATA_FILE=$DIR/dbreport.txt export PROJ_DATA_FILE=$DIR/dbreport.txt
export LAST_APP_DATA_FILE=$DATA_DIR/$APP/data.app.txt export LAST_APP_DATA_FILE=$LAST_DIR/$APP/data.app.txt
export LAST_LOCATION_DATA_FILE=$DATA_DIR/$APP/data.location.txt export LAST_LOCATION_DATA_FILE=$LAST_DIR/$APP/data.location.txt
export LAST_CLIENT_DATA_FILE=$DATA_DIR/$APP/data.client.txt export LAST_CLIENT_DATA_FILE=$LAST_DIR/$APP/data.client.txt
mkdir -p $OUTPUT_DIR/$APP mkdir -p $THIS_DIR/$APP
$DIR/bin/get_data $DIR/bin/get_data
$DIR/bin/metrics > $OUTPUT_DIR/$APP/report.md $DIR/bin/metrics > $THIS_DIR/$APP/report.md
IN_FILE=$OUTPUT_DIR/$APP/report.md IN_FILE=$THIS_DIR/$APP/report.md
OUT_FILE=$OUTPUT_DIR/$APP/$APP-$SHORT_DATE.docx OUT_FILE=$THIS_DIR/$APP/$APP-$SHORT_DATE.docx
} }
function cleanup(){ function cleanup(){
mv $OUTPUT_DIR $DATA_DIR
}
function cleanup2(){
rm -rf $SID_DIR rm -rf $SID_DIR
} }
function main(){ function main(){
DIR=$(dirname $(readlink -f "$0"))
OUTPUT_DIR=$DIR/output
last_month=$(date -d "last month" '+20%y_%m')
export DATA_DIR=$DIR/data/$last_month #data folder must be present
export SID_DIR=$DIR/sid #SID directory
mkdir $SID_DIR
#get user/pass #get user/pass
read -s -p 'User: ' user read -s -p 'User: ' user
echo echo
...@@ -59,7 +47,17 @@ function main(){ ...@@ -59,7 +47,17 @@ function main(){
export USER=$user export USER=$user
export PASS=$pass export PASS=$pass
cleanup DIR=$(dirname $(readlink -f "$0"))
DATA_DIR=$DIR/data
last_month=$(date -d "last month" '+20%y_%m')
this_month=$(date '+20%y_%m')
export THIS_DIR=$DATA_DIR/$this_month
export LAST_DIR=$DATA_DIR/$last_month
export SID_DIR=$DIR/sid #SID directory
mkdir $SID_DIR
start_search 'apps.awesim.org' 'awesim' start_search 'apps.awesim.org' 'awesim'
start_search 'apps.totalsim.us' 'totalsim' start_search 'apps.totalsim.us' 'totalsim'
start_search 'ondemand.osc.edu' 'ondemand' start_search 'ondemand.osc.edu' 'ondemand'
...@@ -73,7 +71,7 @@ function main(){ ...@@ -73,7 +71,7 @@ function main(){
generate_monthly_report 'apps.totalsim.us' 'totalsim' generate_monthly_report 'apps.totalsim.us' 'totalsim'
generate_monthly_report 'ondemand.osc.edu' 'ondemand' generate_monthly_report 'ondemand.osc.edu' 'ondemand'
generate_monthly_report 'stat.osc.edu' 'stats' generate_monthly_report 'stat.osc.edu' 'stats'
cleanup2 cleanup
} }
main main
metrics deleted 100755 → 0
+ 0
− 322
View file @ 9c51aea7
#!/bin/env ruby
#
# Set options here
#
PROJ_FILE = ENV["PROJ_DATA_FILE"] || "dbreport.txt"
APP_FILE = ENV["APP_DATA_FILE"] || "data.app.txt"
CLIENT_FILE = ENV["CLIENT_DATA_FILE"] || "data.client.txt"
LOCATION_FILE = ENV["LOCATION_DATA_FILE"] || "data.location.txt"
SESSION_TIME = (ENV["SESSION_TIME"] || 8).to_i # time in hours between sessions
SHOW_DEV = ENV["SHOW_DEV"] != nil
SHOW_USR = ENV["SHOW_USR"] != nil
SHOW_WEB_VER = ENV["SHOW_WEB_VER"] != nil
SHOW_OS_VER = ENV["SHOW_OS_VER"] != nil
DELIM = "|"
NUM_USR = (ENV["NUM_USR"] || 25).to_i
NUM_WEB = (ENV["NUM_WEB"] || 10).to_i
NUM_OS = (ENV["NUM_OS"] || 10).to_i
NUM_CITY = (ENV["NUM_CITY"] || 10).to_i
APP_TIME_COL = 0
APP_USER_COL = 1
APP_APP_COL = 2
CLNT_TIME_COL = 0
CLNT_USER_COL = 1
CLNT_BRWS_COL = 2
CLNT_BRWSV_COL = 3
CLNT_OS_COL = 4
CLNT_OSV_COL = 5
LOC_TIME_COL = 0
LOC_USER_COL = 1
LOC_CITY_COL = 2
LOC_STATE_COL = 3
LOC_CNTRY_COL = 4
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
require 'securerandom'
require 'time'
require 'csv'
class FormatData
class << self
def simple_table(rows)
lengths = rows.transpose.map {|v| v.map {|x| x.to_s.length}.max}
# print headers
puts wrap(rows[0].zip(lengths).map {|h, l| h.to_s.center(l)}.join(" | "))
puts(
wrap(
lengths.map.with_index do |l, i|
i == 0 ? ("-" * l) : ("-" * (l - 1) + ":")
end.join(" | ")
)
)
# print rows
rows.drop(1).each do |r|
puts(
wrap(
r.zip(lengths).map.with_index do |(v, l), i|
i == 0 ? v.to_s.ljust(l) : v.to_s.rjust(l)
end.join(" | ")
)
)
end
end
def wrap(str)
"| #{str} |"
end
end
end
def generate_user_session(data)
usr_session = {}
data.sort_by { |h| h[:time] }.each do |h|
user = h[:user]
time = h[:time]
if ( sess_info = usr_session[user] ) && ( (time - sess_info[:start_time]) < (SESSION_TIME * 3600) )
sess = sess_info[:sess]
else
sess = SecureRandom.uuid
usr_session[user] = {
sess: sess,
start_time: time
}
end
h[:sess] = sess
end
return usr_session
end
# Read in project data if it exists
projects = CSV.read(PROJ_FILE, headers: true, header_converters: :symbol) if File.exist?(PROJ_FILE)
#fill data with app data
def generate_raw_app_data(app_file)
data = []
File.foreach(app_file) do |line|
line_ary = line.split(DELIM)
time = Time.iso8601(line_ary[APP_TIME_COL]).localtime
user = line_ary[APP_USER_COL]
app = line_ary[APP_APP_COL]
data << {
time: time,
user: user,
app: app
} if (app !~ /^dev/ || SHOW_DEV) && (app !~ /^usr/ || SHOW_USR)
end
return data
end
def generate_app_table_data(data, apps)
app_data = {}
apps.each do |app|
app_ary = data.select {|h| h[:app] == app}
app_data[app] = {
users: app_ary.map {|h| h[:user]}.uniq.size,
launches: app_ary.map {|h| h[:sess]}.uniq.size
}
end
app_data = app_data.sort_by {|k, v| v[:launches]}.reverse
#turn app_data into a table
table_data = []
table_data << ["App", "# Launches", "# Users", "# Launches / User"]
app_data.each do |k, v|
table_data << [k, v[:launches], v[:users], (v[:launches] * 1.0 / v[:users]).round(1)]
end
return table_data
end
data = generate_raw_app_data(APP_FILE)
data.map do |obj|
puts obj
end
# Add session information from collected data
usr_session = generate_user_session(data)
start_time = nil
final_time = nil
start_time = data.map {|h| h[:time]}.min
final_time = data.map {|h| h[:time]}.max
users = data.map {|h| h[:user]}.uniq
user_cnt = users.size
apps = data.map {|h| h[:app]}.uniq
app_cnt = apps.size
app_launches = data.map {|h| [h[:sess], h[:app]]}.uniq.size
title = "Metrics for (#{start_time.strftime("%m/%d/%Y")} - #{final_time.strftime("%m/%d/%Y")})"
puts title
puts "=" * title.length
puts ""
puts "- Total Users = #{user_cnt}"
puts "- Total App Launches = #{app_launches}"
puts "- Avg Launches per User = #{(app_launches * 1.0 / user_cnt).round(1)}"
puts "- Total Used Apps = #{app_cnt}"
title = "Used Apps"
puts ""
puts title
puts "-" * title.length
puts ""
table_data = generate_app_table_data(data, apps)
FormatData.simple_table table_data
top_user = data.map do |obj|
{
key: obj[:user],
sess: obj[:sess],
app: obj[:app]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
num_sessions = v.map { |h| h[:sess] }.uniq.size
num_launches = v.uniq.size
num_apps_used = v.map { |h| h[:app] }.uniq.size
[k, num_sessions, num_launches, (num_launches * 1.0 / num_sessions).round(1), num_apps_used]
end.sort_by { |v| v[1] } .reverse
title = "Top #{NUM_USR} Users"
puts ""
puts title
puts "-" * title.length
puts ""
table_data = []
table_data << ["User", "Name", "# Sessions\\*", "# App Launches", "# Launches / Session", "# Apps Used"]
table_data.last.insert(2, "Institution") if projects
top_user.first(NUM_USR).each do |v|
passwd = Etc.getpwnam(v.first) rescue nil
full_name = passwd ? passwd.gecos : "Unknown"
v.insert(1, full_name)
if projects
group = passwd && Etc.getgrgid(passwd.gid) rescue nil
primary_group = group ? group.name : "Unknown"
institution = if row = projects.find { |row| row[:project] == primary_group }
row[:institution_name]
else
primary_group
end
v.insert(2, institution)
end
table_data << v
end
FormatData.simple_table table_data
data = []
File.foreach(CLIENT_FILE) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[CLNT_USER_COL],
time: Time.iso8601(line_ary[CLNT_TIME_COL]).localtime,
browser: line_ary[CLNT_BRWS_COL],
browser_v: line_ary[CLNT_BRWSV_COL],
os: line_ary[CLNT_OS_COL],
os_v: line_ary[CLNT_OSV_COL]
}
end
# Add session information from collected data
usr_session = generate_user_session(data)
top_browser = data.map do |obj|
{
key: SHOW_WEB_VER ? "#{obj[:browser]} #{obj[:browser_v].to_i}" : obj[:browser],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
top_os = data.map do |obj|
{
key: SHOW_OS_VER ? "#{obj[:os]} #{obj[:os_v]}" : obj[:os],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
title = "Top #{NUM_WEB} Browsers"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Browsers = #{top_browser.size}"
puts ""
table_data = []
table_data << ["Browser", "# Sessions\\*", "# Users"]
top_browser.first(NUM_WEB).each do |v|
table_data << v
end
FormatData.simple_table table_data
title = "Top #{NUM_OS} Operating Systems"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Operating Systems = #{top_os.size}"
puts ""
table_data = []
table_data << ["OS", "# Sessions\\*", "# Users"]
top_os.first(NUM_OS).each do |v|
table_data << v
end
FormatData.simple_table table_data
data = []
File.foreach(LOCATION_FILE) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[LOC_USER_COL],
time: Time.iso8601(line_ary[LOC_TIME_COL]).localtime,
city: line_ary[LOC_CITY_COL],
state: line_ary[LOC_STATE_COL],
country: line_ary[LOC_CNTRY_COL],
}
end
# Add session information from collected data
usr_session = generate_user_session(data)
top_location = data.map do |obj|
{
key: "#{obj[:city]}, #{obj[:state]}, #{obj[:country]}",
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
title = "Top #{NUM_CITY} Cities"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Cities = #{top_location.size}"
puts ""
table_data = []
table_data << ["City", "# Sessions\\*", "# Users"]
top_location.first(NUM_CITY).each do |v|
table_data << v
end
FormatData.simple_table table_data
puts ""
puts "\\* Session = All activity within the #{SESSION_TIME} hour period of time from first user activity."
metricsTest.sh deleted 100644 → 0
+ 0
− 17
View file @ 9c51aea7
DIR=$(dirname $(readlink -f "$0"))
OUTPUT_DIR=$DIR/testText
last_month=$(date -d "last month" '+20%y_%m')
export DATA_DIR=$DIR/data/$last_month #data folder must be present
export APP_DATA_FILE=$OUTPUT_DIR/data.app.txt
export LOCATION_DATA_FILE=$OUTPUT_DIR/data.location.txt
export CLIENT_DATA_FILE=$OUTPUT_DIR/data.client.txt
export PROJ_DATA_FILE=$DIR/dbreport.txt
export LAST_APP_DATA_FILE=$DATA_DIR/ondemand/data.app.txt
export LAST_LOCATION_DATA_FILE=$DATA_DIR/ondemand/data.location.txt
export LAST_CLIENT_DATA_FILE=$DATA_DIR/ondemand/data.client.txt
$DIR/bin/metricsTest > $DIR/report.md
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment