Skip to content
Snippets Groups Projects
Commit 44ea926d authored by matthu017's avatar matthu017
Browse files

Moved classes to lib 

Changed data file structure - preset data
parent fa146d87
No related branches found
No related tags found
1 merge request!2Refactored to use splunk
Hide whitespace changes
Inline Side-by-side
Showing
with 747 additions and 540 deletions
.gitignore
+ 1
− 1
View file @ 44ea926d
...@@ -53,4 +53,4 @@ build-iPhoneSimulator/ ...@@ -53,4 +53,4 @@ build-iPhoneSimulator/
google_api_key.json google_api_key.json
# Ignore metrics data # Ignore metrics data
/output/
bin/get_app_data
+ 10
− 9
View file @ 44ea926d
#!/bin/env ruby #!/bin/env ruby
#
# Set dependencies here
#
require 'net/http' require 'net/http'
require 'uri' require 'uri'
require 'csv' require 'csv'
require 'openssl' require 'openssl'
require 'rexml/document' require 'rexml/document'
include REXML include REXML
require 'rubygems'
require 'bundler/setup'
# #
# Set options here # Set options here
# #
OUTPUT = ENV["APP_DATA_FILE"] || "data.app.txt"
SID_DIR = ENV["SID_DIR"] || "sid"
APP = ENV["APP"] || "ondemand" APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu" HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29" START_DATE = ENV["START_DATE"] || "2016-11-29"
...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today" ...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin" USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass" PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{DIR}/output/current/#{APP}/data.app.txt"
COUNT = 60000 COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089' BASEURL = 'https://splunk.osc.edu:8089'
DELIM = "|" DELIM = "|"
...@@ -82,13 +90,6 @@ APP_TOKEN = { ...@@ -82,13 +90,6 @@ APP_TOKEN = {
%r{^/dev/(?<app_name>[^/]+)} => "dev/%{user}/%{app_name}" %r{^/dev/(?<app_name>[^/]+)} => "dev/%{user}/%{app_name}"
} }
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
STDERR.puts "APP_DATA_FILE = #{OUTPUT}" STDERR.puts "APP_DATA_FILE = #{OUTPUT}"
STDERR.puts "HOST = #{HOST}" STDERR.puts "HOST = #{HOST}"
STDERR.puts "START_DATE = #{START_DATE}" STDERR.puts "START_DATE = #{START_DATE}"
......
bin/get_client_data
+ 10
− 8
View file @ 44ea926d
#!/bin/env ruby #!/bin/env ruby
#
# Set dependencies here
#
require 'net/http' require 'net/http'
require 'uri' require 'uri'
require 'csv' require 'csv'
require 'openssl' require 'openssl'
require 'rexml/document' require 'rexml/document'
include REXML include REXML
require 'rubygems'
require 'bundler/setup'
# #
# Set options here # Set options here
# #
OUTPUT = ENV["CLIENT_DATA_FILE"] || "data.client.txt"
SID_DIR = ENV["SID_DIR"] || "sid"
APP = ENV["APP"] || "ondemand" APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu" HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29" START_DATE = ENV["START_DATE"] || "2016-11-29"
...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today" ...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin" USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass" PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{DIR}/output/current/#{APP}/data.client.txt"
COUNT = 60000 COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089' BASEURL = 'https://splunk.osc.edu:8089'
DELIM = "|" DELIM = "|"
...@@ -29,12 +37,6 @@ OS_VERSION = 5 ...@@ -29,12 +37,6 @@ OS_VERSION = 5
OS_MINOR = 8 OS_MINOR = 8
OS_PATCH = 9 OS_PATCH = 9
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
STDERR.puts "CLIENT_DATA_FILE = #{OUTPUT}" STDERR.puts "CLIENT_DATA_FILE = #{OUTPUT}"
STDERR.puts "HOST = #{HOST}" STDERR.puts "HOST = #{HOST}"
......
bin/get_location_data
+ 10
− 4
View file @ 44ea926d
#!/bin/env ruby #!/bin/env ruby
#
# Set dependencies here
#
require 'net/http' require 'net/http'
require 'uri' require 'uri'
require 'csv' require 'csv'
require 'openssl' require 'openssl'
require 'rexml/document' require 'rexml/document'
include REXML include REXML
require 'rubygems'
require 'bundler/setup'
# #
# Set options here # Set options here
# #
OUTPUT = ENV["LOCATION_DATA_FILE"] || "data.location.txt"
SID_DIR = ENV["SID_DIR"] || "sid"
APP = ENV["APP"] || "ondemand" APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu" HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29" START_DATE = ENV["START_DATE"] || "2016-11-29"
...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today" ...@@ -18,6 +22,10 @@ END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin" USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass" PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{DIR}/output/current/#{APP}/data.location.txt"
COUNT = 60000 COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089' BASEURL = 'https://splunk.osc.edu:8089'
DELIM = "|" DELIM = "|"
...@@ -109,6 +117,4 @@ ensure ...@@ -109,6 +117,4 @@ ensure
target.close target.close
end end
gather_data(OUTPUT) gather_data(OUTPUT)
bin/metrics
+ 31
− 492
View file @ 44ea926d
...@@ -3,19 +3,35 @@ ...@@ -3,19 +3,35 @@
# #
# Set options here # Set options here
# #
PROJ_FILE = ENV["PROJ_DATA_FILE"] || "dbreport.txt" APP = ENV["APP"] || "ondemand"
APP_FILE = ENV["APP_DATA_FILE"] || "data.app.txt" HOST = ENV["HOST"] || "ondemand.osc.edu"
CLIENT_FILE = ENV["CLIENT_DATA_FILE"] || "data.client.txt"
LOCATION_FILE = ENV["LOCATION_DATA_FILE"] || "data.location.txt" DIR = ENV["DIR"] || "./"
LAST_APP_FILE = ENV["LAST_APP_DATA_FILE"] || "data.app.txt" CURRENT = "#{DIR}/output/current"
LAST_CLIENT_FILE = ENV["LAST_CLIENT_DATA_FILE"] || "data.client.txt" PREVIOUS = "#{DIR}/output/previous"
LAST_LOCATION_FILE = ENV["LAST_LOCATION_DATA_FILE"]|| "data.location.txt" LIB = "#{DIR}/lib"
PROJ_FILE = "#{DIR}/dbreport.txt"
APP_FILE = "#{CURRENT}/#{APP}/data.app.txt"
CLIENT_FILE = "#{CURRENT}/#{APP}/data.client.txt"
LOCATION_FILE = "#{CURRENT}/#{APP}/data.location.txt"
LAST_APP_FILE = "#{PREVIOUS}/#{APP}/data.app.txt"
LAST_CLIENT_FILE = "#{PREVIOUS}/#{APP}/data.client.txt"
LAST_LOCATION_FILE = "#{PREVIOUS}/#{APP}/data.location.txt"
#
# Set dependencies here
#
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "lib"))
require "#{LIB}/AppData"
require "#{LIB}/ClientData"
require "#{LIB}/LocationData"
require "#{LIB}/DataMethods"
require 'rubygems'
require 'bundler/setup'
SESSION_TIME = (ENV["SESSION_TIME"] || 8).to_i # time in hours between sessions SESSION_TIME = (ENV["SESSION_TIME"] || 8).to_i # time in hours between sessions
SHOW_DEV = ENV["SHOW_DEV"] != nil
SHOW_USR = ENV["SHOW_USR"] != nil
SHOW_WEB_VER = ENV["SHOW_WEB_VER"] != nil
SHOW_OS_VER = ENV["SHOW_OS_VER"] != nil
DELIM = "|" DELIM = "|"
NUM_USR = (ENV["NUM_USR"] || 25).to_i NUM_USR = (ENV["NUM_USR"] || 25).to_i
...@@ -23,34 +39,6 @@ NUM_WEB = (ENV["NUM_WEB"] || 10).to_i ...@@ -23,34 +39,6 @@ NUM_WEB = (ENV["NUM_WEB"] || 10).to_i
NUM_OS = (ENV["NUM_OS"] || 10).to_i NUM_OS = (ENV["NUM_OS"] || 10).to_i
NUM_CITY = (ENV["NUM_CITY"] || 10).to_i NUM_CITY = (ENV["NUM_CITY"] || 10).to_i
APP_TIME_COL = 0
APP_USER_COL = 1
APP_APP_COL = 2
CLNT_TIME_COL = 0
CLNT_USER_COL = 1
CLNT_BRWS_COL = 2
CLNT_BRWSV_COL = 3
CLNT_OS_COL = 4
CLNT_OSV_COL = 5
LOC_TIME_COL = 0
LOC_USER_COL = 1
LOC_CITY_COL = 2
LOC_STATE_COL = 3
LOC_CNTRY_COL = 4
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
require 'securerandom'
require 'time'
require 'csv'
class FormatData class FormatData
class << self class << self
def simple_table(rows) def simple_table(rows)
...@@ -82,460 +70,11 @@ class FormatData ...@@ -82,460 +70,11 @@ class FormatData
end end
end end
class Data_Methods
def return_percent_change(this_month, last_month)
value = (((this_month.to_f / last_month) -1) * 100).round(1)
returnVal = "0.0%"
if (value > 0)
returnVal = "(+#{value}%)"
else
returnVal = "(#{value}%)"
end
return returnVal
end
private
def generate_user_session(data)
usr_session = {}
data.sort_by { |h| h[:time] }.each do |h|
user = h[:user]
time = h[:time]
if ( sess_info = usr_session[user] ) && ( (time - sess_info[:start_time]) < (SESSION_TIME * 3600) )
sess = sess_info[:sess]
else
sess = SecureRandom.uuid
usr_session[user] = {
sess: sess,
start_time: time
}
end
h[:sess] = sess
end
end
def return_rank_change(value)
returnVal = "(-)"
if (value > 0)
returnVal = "(+#{value})"
elsif (value < 0)
returnVal = "(#{value})"
end
return returnVal
end
end
class App_Data < Data_Methods
def initialize(app_file, project_file)
@data = generate_raw_app_data(app_file)
generate_user_session(@data)
@start_time = @data.map {|h| h[:time]}.min
@final_time = @data.map {|h| h[:time]}.max
@users = @data.map {|h| h[:user]}.uniq
@user_cnt = @users.size
@apps = @data.map {|h| h[:app]}.uniq
@app_cnt = @apps.size
@app_launches = @data.map {|h| [h[:sess], h[:app]]}.uniq.size
@avg_launches = (@app_launches * 1.0 / @user_cnt).round(1)
@app_table = generate_app_table(@data, @apps)
@user_table = generate_user_table(@data, project_file)
end
attr_reader :start_time, :final_time, :user_cnt, :app_cnt, :app_launches, :avg_launches, :app_table, :user_table
#returns app data in comparison to previous month
def format_app_table(last_month_app_data)
table_data = []
table_data << @app_table.shift
#need to skip the first line...
@app_table.each do |this_month_app|
last_month_app = last_month_app_data.return_app_usage(this_month_app[1])
rank_change = "(-)"
launch_change = "(-)"
users_change = "(-)"
lu_change = "(-)"
if (last_month_app != "N/A")
rank_change = return_rank_change(last_month_app[0] - this_month_app[0])
launch_change = return_percent_change(this_month_app[2], last_month_app[2])
users_change = return_percent_change(this_month_app[3], last_month_app[3])
lu_change = return_percent_change(this_month_app[4], last_month_app[4])
end
#push to table data
table_data << ["#{this_month_app[0]} #{rank_change}",
"#{this_month_app[1]}",
"#{this_month_app[2]} #{launch_change}",
"#{this_month_app[3]} #{users_change}",
"#{this_month_app[4]} #{lu_change}"
]
end
return table_data
end
def format_user_table(last_month_app_data)
table_data = []
table_data << @user_table.shift
@user_table.each do |this_month_user|
last_month_user = last_month_app_data.return_user_usage(this_month_user[1])
rank_change = "(-)"
session_change = "(-)"
app_launch_change = "(-)"
launches_session_change = "(-)"
apps_used = "(-)"
if (last_month_user != "N/A")
rank_change = return_rank_change(last_month_user[0] - this_month_user[0])
session_change = return_percent_change(this_month_user[4], last_month_user[4])
app_launch_change = return_percent_change(this_month_user[5], last_month_user[5])
launches_session_change = return_percent_change(this_month_user[6], last_month_user[6])
apps_used = return_percent_change(this_month_user[7], last_month_user[7])
end
table_data << ["#{this_month_user[0]} #{rank_change}",
"#{this_month_user[1]}",
"#{this_month_user[2]}",
"#{this_month_user[3]}",
"#{this_month_user[4]} #{session_change}",
"#{this_month_user[5]} #{app_launch_change}",
"#{this_month_user[6]} #{launches_session_change}",
"#{this_month_user[7]} #{apps_used}"
]
end
return table_data.first(NUM_USR+1) #add one for the header
end
def return_app_usage(app_name)
@app_table.each do |app|
if (app[1] == app_name)
return app
end
end
return "N/A"
end
def return_user_usage(username)
@user_table.each do |user|
if (user[1] == username)
return user
end
end
return "N/A"
end
private
#fill data with app data
def generate_raw_app_data(app_file)
data = []
File.foreach(app_file) do |line|
line_ary = line.split(DELIM)
time = Time.iso8601(line_ary[APP_TIME_COL]).localtime
user = line_ary[APP_USER_COL]
app = line_ary[APP_APP_COL]
data << {
time: time,
user: user,
app: app
} if (app !~ /^dev/ || SHOW_DEV) && (app !~ /^usr/ || SHOW_USR)
end
return data
end
def generate_app_table(data, apps)
app_data = {}
apps.each do |app|
app_ary = data.select {|h| h[:app] == app}
app_data[app] = {
users: app_ary.map {|h| h[:user]}.uniq.size,
launches: app_ary.map {|h| h[:sess]}.uniq.size
}
end
app_data = app_data.sort_by {|k, v| v[:launches]}.reverse
#turn app_data into an array, rank is assigned based on sessions, then app launches
table_data = []
table_data << ["Rank", "App", "# Launches", "# Users", "# Launches / User"]
rank = 1
app_data.each do |k, v|
table_data << [rank, k, v[:launches], v[:users], (v[:launches] * 1.0 / v[:users]).round(1)]
rank += 1
end
return table_data
end
def generate_user_table(data, project_file)
projects = CSV.read(project_file, headers: true, header_converters: :symbol) if File.exist?(project_file)
top_user = data.map do |obj|
{
key: obj[:user],
sess: obj[:sess],
app: obj[:app]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
num_sessions = v.map { |h| h[:sess] }.uniq.size
num_launches = v.uniq.size
num_apps_used = v.map { |h| h[:app] }.uniq.size
[k, num_sessions, num_launches, (num_launches * 1.0 / num_sessions).round(1), num_apps_used]
end.sort_by { |v| v[1] } .reverse
ranking = 1
table_data = []
table_data << ["Rank", "User", "Name", "# Sessions\\*", "# App Launches", "# Launches / Session", "# Apps Used"]
table_data.last.insert(3, "Institution") if projects #insert at 3, ranking will shift all values
top_user.each do |v|
passwd = Etc.getpwnam(v.first) rescue nil
full_name = passwd ? passwd.gecos : "Unknown"
v.insert(1, full_name)
if projects
group = passwd && Etc.getgrgid(passwd.gid) rescue nil
primary_group = group ? group.name : "Unknown"
institution = if row = projects.find { |row| row[:project] == primary_group }
row[:institution_name]
else
primary_group
end
v.insert(2, institution)
v.insert(0, ranking)
ranking += 1
end
table_data << v
end
return table_data
end
end
class Client_Data < Data_Methods
def initialize(client_file)
@data = generate_raw_client_data(client_file)
generate_user_session(@data)
@browser_table = generate_browser_table(@data)
@os_table = generate_os_table(@data)
end
attr_reader :browser_table, :os_table
def format_browser_table(last_month_client_data)
table_data = []
table_data << @browser_table.shift
@browser_table.each do |this_month_browser|
last_month_browser = last_month_client_data.return_browser_usage(this_month_browser[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_browser != "N/A")
rank_change = return_rank_change (last_month_browser[0] - this_month_browser[0])
session_change = return_percent_change(this_month_browser[2], last_month_browser[2])
users_change = return_percent_change(this_month_browser[3], last_month_browser[3])
end
table_data << ["#{this_month_browser[0]} #{rank_change}",
"#{this_month_browser[1]}",
"#{this_month_browser[2]} #{session_change}",
"#{this_month_browser[3]} #{users_change}",
]
end
return table_data.first(NUM_WEB + 1) #because of the header
end
def format_os_table(last_month_client_data)
table_data = []
table_data << @os_table.shift
@os_table.each do |this_month_os|
last_month_os = last_month_client_data.return_os_usage(this_month_os[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_os != "N/A")
rank_change = return_rank_change (last_month_os[0] - this_month_os[0])
session_change = return_percent_change(this_month_os[2], last_month_os[2])
users_change = return_percent_change(this_month_os[3], last_month_os[3])
end
table_data << ["#{this_month_os[0]} #{rank_change}",
"#{this_month_os[1]}",
"#{this_month_os[2]} #{session_change}",
"#{this_month_os[3]} #{users_change}",
]
end
return table_data.first(NUM_OS + 1) #because of the header
end
def return_browser_usage(client_name)
@browser_table.each do |browser|
if (browser[1] == client_name)
return browser
end
end
return "N/A"
end
def return_os_usage(os_name)
@os_table.each do |os|
if (os[1] == os_name)
return os
end
end
return "N/A"
end
private
def generate_raw_client_data(client_file)
data = []
File.foreach(client_file) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[CLNT_USER_COL],
time: Time.iso8601(line_ary[CLNT_TIME_COL]).localtime,
browser: line_ary[CLNT_BRWS_COL],
browser_v: line_ary[CLNT_BRWSV_COL],
os: line_ary[CLNT_OS_COL],
os_v: line_ary[CLNT_OSV_COL]
}
end
return data
end
def generate_browser_table(data)
top_browser = data.map do |obj|
{
key: SHOW_WEB_VER ? "#{obj[:browser]} #{obj[:browser_v].to_i}" : obj[:browser],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_browser.map do |v|
v.unshift(rank)
rank += 1
end
top_browser.unshift(["Rank", "Browser", "# Sessions\\*", "# Users"])
return top_browser
end
def generate_os_table(data)
top_os = data.map do |obj|
{
key: SHOW_OS_VER ? "#{obj[:os]} #{obj[:os_v]}" : obj[:os],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_os.map do |v|
v.unshift(rank)
rank += 1
end
top_os.unshift(["Rank", "OS", "# Sessions\\*", "# Users"])
return top_os
end
end
class Location_Data < Data_Methods
def initialize(location_file)
@data = generate_raw_location_data(location_file)
generate_user_session(@data)
@location_table = generate_location_table(@data)
end
attr_reader :location_table
def format_location_table(last_month_location_data)
table_data = []
table_data << @location_table.shift
@location_table.each do |this_month_location|
last_month_location = last_month_location_data.return_location_usage(this_month_location[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_location != "N/A")
rank_change = return_rank_change (last_month_location[0] - this_month_location[0])
session_change = return_percent_change(this_month_location[2], last_month_location[2])
users_change = return_percent_change(this_month_location[3], last_month_location[3])
end
table_data << ["#{this_month_location[0]} #{rank_change}",
"#{this_month_location[1]}",
"#{this_month_location[2]} #{session_change}",
"#{this_month_location[3]} #{users_change}",
]
end
return table_data.first(NUM_CITY + 1) #because of the header
end
def return_location_usage(location_name)
@location_table.each do |location|
if (location[1] == location_name)
return location
end
end
return "N/A"
end
private
def generate_raw_location_data(location_file)
data = []
File.foreach(location_file) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[LOC_USER_COL],
time: Time.iso8601(line_ary[LOC_TIME_COL]).localtime,
city: line_ary[LOC_CITY_COL],
state: line_ary[LOC_STATE_COL],
country: line_ary[LOC_CNTRY_COL],
}
end
return data #array
end
def generate_location_table(data)
top_location = data.map do |obj|
{
key: "#{obj[:city]}, #{obj[:state]}, #{obj[:country]}",
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_location.map do |v|
v.unshift(rank)
rank += 1
end
top_location.unshift(["Rank", "City", "# Sessions\\*", "# Users"])
return top_location #array
end
end
class Report class Report
def initialize (app_file, client_file, location_file, project_file) def initialize (app_file, client_file, location_file, project_file)
@app = App_Data.new(app_file, project_file) @app = AppData.new(app_file, project_file)
@client = Client_Data.new(client_file) @client = ClientData.new(client_file)
@location = Location_Data.new(location_file) @location = LocationData.new(location_file)
end end
attr_reader :app, :client, :location attr_reader :app, :client, :location
end end
...@@ -543,7 +82,7 @@ end ...@@ -543,7 +82,7 @@ end
def main def main
report1 = Report.new(APP_FILE, CLIENT_FILE, LOCATION_FILE, PROJ_FILE) report1 = Report.new(APP_FILE, CLIENT_FILE, LOCATION_FILE, PROJ_FILE)
report2 = Report.new(LAST_APP_FILE, LAST_CLIENT_FILE, LAST_LOCATION_FILE, PROJ_FILE) report2 = Report.new(LAST_APP_FILE, LAST_CLIENT_FILE, LAST_LOCATION_FILE, PROJ_FILE)
compare = Data_Methods.new compare = DataMethods.new
totalUser = compare.return_percent_change(report1.app.user_cnt, report2.app.user_cnt) totalUser = compare.return_percent_change(report1.app.user_cnt, report2.app.user_cnt)
totalApp = compare.return_percent_change(report1.app.app_launches, report2.app.app_launches) totalApp = compare.return_percent_change(report1.app.app_launches, report2.app.app_launches)
avgLaunch = compare.return_percent_change(report1.app.avg_launches, report2.app.avg_launches) avgLaunch = compare.return_percent_change(report1.app.avg_launches, report2.app.avg_launches)
......
generate_reports.sh
+ 11
− 26
View file @ 44ea926d
...@@ -17,25 +17,14 @@ function generate_monthly_report(){ ...@@ -17,25 +17,14 @@ function generate_monthly_report(){
export END_DATE=$(date +%Y-%m-%d) export END_DATE=$(date +%Y-%m-%d)
SHORT_DATE=$(date +%Y-%m) SHORT_DATE=$(date +%Y-%m)
export APP_DATA_FILE=$THIS_DIR/$APP/data.app.txt CURRENT_DIR=$DIR/output/current/$APP
export LOCATION_DATA_FILE=$THIS_DIR/$APP/data.location.txt mkdir $CURRENT_DIR
export CLIENT_DATA_FILE=$THIS_DIR/$APP/data.client.txt
export PROJ_DATA_FILE=$DIR/dbreport.txt
export LAST_APP_DATA_FILE=$LAST_DIR/$APP/data.app.txt
export LAST_LOCATION_DATA_FILE=$LAST_DIR/$APP/data.location.txt
export LAST_CLIENT_DATA_FILE=$LAST_DIR/$APP/data.client.txt
mkdir -p $THIS_DIR/$APP
$DIR/bin/get_data $DIR/bin/get_data
$DIR/bin/metrics > $THIS_DIR/$APP/report.md $DIR/bin/metrics > $CURRENT_DIR/report.md
IN_FILE=$THIS_DIR/$APP/report.md
OUT_FILE=$THIS_DIR/$APP/$APP-$SHORT_DATE.docx
}
function cleanup(){ IN_FILE=$CURRENT_DIR/report.md
rm -rf $SID_DIR OUT_FILE=$CURRENT_DIR/$APP-$SHORT_DATE.docx
} }
function main(){ function main(){
...@@ -47,17 +36,13 @@ function main(){ ...@@ -47,17 +36,13 @@ function main(){
export USER=$user export USER=$user
export PASS=$pass export PASS=$pass
DIR=$(dirname $(readlink -f "$0")) export DIR=$(dirname $(readlink -f "$0"))
DATA_DIR=$DIR/data SID_DIR=$DIR/sid #SID directory
last_month=$(date -d "last month" '+20%y_%m')
this_month=$(date '+20%y_%m')
export THIS_DIR=$DATA_DIR/$this_month
export LAST_DIR=$DATA_DIR/$last_month
export SID_DIR=$DIR/sid #SID directory
mkdir $SID_DIR mkdir $SID_DIR
rm -rf $DIR/output/previous
mv $DIR/output/current $DIR/output/previous
start_search 'apps.awesim.org' 'awesim' start_search 'apps.awesim.org' 'awesim'
start_search 'apps.totalsim.us' 'totalsim' start_search 'apps.totalsim.us' 'totalsim'
start_search 'ondemand.osc.edu' 'ondemand' start_search 'ondemand.osc.edu' 'ondemand'
...@@ -71,7 +56,7 @@ function main(){ ...@@ -71,7 +56,7 @@ function main(){
generate_monthly_report 'apps.totalsim.us' 'totalsim' generate_monthly_report 'apps.totalsim.us' 'totalsim'
generate_monthly_report 'ondemand.osc.edu' 'ondemand' generate_monthly_report 'ondemand.osc.edu' 'ondemand'
generate_monthly_report 'stat.osc.edu' 'stats' generate_monthly_report 'stat.osc.edu' 'stats'
cleanup rm -rf $SID_DIR
} }
main main
lib/.nfs00000000000e48c200000001 0 → 100755
+ 166
− 0
View file @ 44ea926d
#!/bin/env ruby
require './lib/AppData'
require './lib/ClientData'
require './lib/LocationData'
require './lib/DataMethods'
#
# Set options here
#
PROJ_FILE = ENV["PROJ_DATA_FILE"] || "dbreport.txt"
APP_FILE = ENV["APP_DATA_FILE"] || "data.app.txt"
CLIENT_FILE = ENV["CLIENT_DATA_FILE"] || "data.client.txt"
LOCATION_FILE = ENV["LOCATION_DATA_FILE"] || "data.location.txt"
LAST_APP_FILE = ENV["LAST_APP_DATA_FILE"] || "data.app.txt"
LAST_CLIENT_FILE = ENV["LAST_CLIENT_DATA_FILE"] || "data.client.txt"
LAST_LOCATION_FILE = ENV["LAST_LOCATION_DATA_FILE"]|| "data.location.txt"
SESSION_TIME = (ENV["SESSION_TIME"] || 8).to_i # time in hours between sessions
SHOW_DEV = ENV["SHOW_DEV"] != nil
SHOW_USR = ENV["SHOW_USR"] != nil
SHOW_WEB_VER = ENV["SHOW_WEB_VER"] != nil
SHOW_OS_VER = ENV["SHOW_OS_VER"] != nil
DELIM = "|"
NUM_USR = (ENV["NUM_USR"] || 25).to_i
NUM_WEB = (ENV["NUM_WEB"] || 10).to_i
NUM_OS = (ENV["NUM_OS"] || 10).to_i
NUM_CITY = (ENV["NUM_CITY"] || 10).to_i
APP_TIME_COL = 0
APP_USER_COL = 1
APP_APP_COL = 2
CLNT_TIME_COL = 0
CLNT_USER_COL = 1
CLNT_BRWS_COL = 2
CLNT_BRWSV_COL = 3
CLNT_OS_COL = 4
CLNT_OSV_COL = 5
LOC_TIME_COL = 0
LOC_USER_COL = 1
LOC_CITY_COL = 2
LOC_STATE_COL = 3
LOC_CNTRY_COL = 4
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
require 'securerandom'
require 'time'
require 'csv'
class FormatData
class << self
def simple_table(rows)
lengths = rows.transpose.map {|v| v.map {|x| x.to_s.length}.max}
# print headers
puts wrap(rows[0].zip(lengths).map {|h, l| h.to_s.center(l)}.join(" | "))
puts(
wrap(
lengths.map.with_index do |l, i|
i == 0 ? ("-" * l) : ("-" * (l - 1) + ":")
end.join(" | ")
)
)
# print rows
rows.drop(1).each do |r|
puts(
wrap(
r.zip(lengths).map.with_index do |(v, l), i|
i == 0 ? v.to_s.ljust(l) : v.to_s.rjust(l)
end.join(" | ")
)
)
end
end
def wrap(str)
"| #{str} |"
end
end
end
class Report
def initialize (app_file, client_file, location_file, project_file)
@app = AppData.new(app_file, project_file)
@client = ClientData.new(client_file)
@location = LocationData.new(location_file)
end
attr_reader :app, :client, :location
end
def main
report1 = Report.new(APP_FILE, CLIENT_FILE, LOCATION_FILE, PROJ_FILE)
report2 = Report.new(LAST_APP_FILE, LAST_CLIENT_FILE, LAST_LOCATION_FILE, PROJ_FILE)
compare = DataMethods.new
totalUser = compare.return_percent_change(report1.app.user_cnt, report2.app.user_cnt)
totalApp = compare.return_percent_change(report1.app.app_launches, report2.app.app_launches)
avgLaunch = compare.return_percent_change(report1.app.avg_launches, report2.app.avg_launches)
totalUsed = compare.return_percent_change(report1.app.app_cnt, report2.app.app_cnt)
title = "Metrics for (#{report1.app.start_time.strftime("%m/%d/%Y")} - #{report1.app.final_time.strftime("%m/%d/%Y")}) "
subtitle = "Compared to (#{report2.app.start_time.strftime("%m/%d/%Y")} - #{report2.app.final_time.strftime("%m/%d/%Y")})"
puts title + subtitle
puts "=" * (title.length + subtitle.length)
puts ""
puts "- Total Users = #{report1.app.user_cnt} #{totalUser}"
puts "- Total App Launches = #{report1.app.app_launches} #{totalApp}"
puts "- Avg Launches per User = #{report1.app.avg_launches} #{avgLaunch}"
puts "- Total Used Apps = #{report1.app.app_cnt} #{totalUsed}"
title = "Used Apps"
puts ""
puts title
puts "-" * title.length
puts ""
FormatData.simple_table report1.app.format_app_table(report2.app)
title = "Top #{NUM_USR} Users"
puts ""
puts title
puts "-" * title.length
puts ""
FormatData.simple_table report1.app.format_user_table(report2.app)
title = "Top #{NUM_WEB} Browsers"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Browsers = #{report1.client.browser_table.size - 1}" #title block
puts ""
FormatData.simple_table report1.client.format_browser_table(report2.client)
title = "Top #{NUM_OS} Operating Systems"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Operating Systems = #{report1.client.os_table.size - 1}" #title block
puts ""
FormatData.simple_table report1.client.format_os_table(report2.client)
title = "Top #{NUM_CITY} Cities"
puts ""
puts title
puts "-" * title.length
puts ""
puts "- Total Cities = #{report1.location.location_table.size - 1}" #title block
puts ""
FormatData.simple_table report1.location.format_location_table(report2.location)
puts ""
puts "\\* Session = All activity within the #{SESSION_TIME} hour period of time from first user activity."
end
main
lib/AppData.rb 0 → 100644
+ 208
− 0
View file @ 44ea926d
#!/bin/env ruby
#
# Set options here
#
SHOW_DEV = ENV["SHOW_DEV"] != nil
SHOW_USR = ENV["SHOW_USR"] != nil
APP_TIME_COL = 0
APP_USER_COL = 1
APP_APP_COL = 2
#
# Set dependencies here
#
require 'time'
require 'csv'
require "#{LIB}/DataMethods"
class AppData < DataMethods
def initialize(app_file, project_file)
@data = generate_raw_app_data(app_file)
generate_user_session(@data)
@start_time = @data.map {|h| h[:time]}.min
@final_time = @data.map {|h| h[:time]}.max
@users = @data.map {|h| h[:user]}.uniq
@user_cnt = @users.size
@apps = @data.map {|h| h[:app]}.uniq
@app_cnt = @apps.size
@app_launches = @data.map {|h| [h[:sess], h[:app]]}.uniq.size
@avg_launches = (@app_launches * 1.0 / @user_cnt).round(1)
@app_table = generate_app_table(@data, @apps)
@user_table = generate_user_table(@data, project_file)
end
attr_reader :start_time, :final_time, :user_cnt, :app_cnt, :app_launches, :avg_launches, :app_table, :user_table
#returns app data in comparison to previous month
def format_app_table(last_month_app_data)
table_data = []
table_data << @app_table.shift
#need to skip the first line...
@app_table.each do |this_month_app|
last_month_app = last_month_app_data.return_app_usage(this_month_app[1])
rank_change = "(-)"
launch_change = "(-)"
users_change = "(-)"
lu_change = "(-)"
if (last_month_app != "N/A")
rank_change = return_rank_change(last_month_app[0] - this_month_app[0])
launch_change = return_percent_change(this_month_app[2], last_month_app[2])
users_change = return_percent_change(this_month_app[3], last_month_app[3])
lu_change = return_percent_change(this_month_app[4], last_month_app[4])
end
#push to table data
table_data << ["#{this_month_app[0]} #{rank_change}",
"#{this_month_app[1]}",
"#{this_month_app[2]} #{launch_change}",
"#{this_month_app[3]} #{users_change}",
"#{this_month_app[4]} #{lu_change}"
]
end
return table_data
end
def format_user_table(last_month_app_data)
table_data = []
table_data << @user_table.shift
@user_table.each do |this_month_user|
last_month_user = last_month_app_data.return_user_usage(this_month_user[1])
rank_change = "(-)"
session_change = "(-)"
app_launch_change = "(-)"
launches_session_change = "(-)"
apps_used = "(-)"
if (last_month_user != "N/A")
rank_change = return_rank_change(last_month_user[0] - this_month_user[0])
session_change = return_percent_change(this_month_user[4], last_month_user[4])
app_launch_change = return_percent_change(this_month_user[5], last_month_user[5])
launches_session_change = return_percent_change(this_month_user[6], last_month_user[6])
apps_used = return_percent_change(this_month_user[7], last_month_user[7])
end
table_data << ["#{this_month_user[0]} #{rank_change}",
"#{this_month_user[1]}",
"#{this_month_user[2]}",
"#{this_month_user[3]}",
"#{this_month_user[4]} #{session_change}",
"#{this_month_user[5]} #{app_launch_change}",
"#{this_month_user[6]} #{launches_session_change}",
"#{this_month_user[7]} #{apps_used}"
]
end
return table_data.first(NUM_USR+1) #add one for the header
end
def return_app_usage(app_name)
@app_table.each do |app|
if (app[1] == app_name)
return app
end
end
return "N/A"
end
def return_user_usage(username)
@user_table.each do |user|
if (user[1] == username)
return user
end
end
return "N/A"
end
private
#fill data with app data
def generate_raw_app_data(app_file)
data = []
File.foreach(app_file) do |line|
line_ary = line.split(DELIM)
time = Time.iso8601(line_ary[APP_TIME_COL]).localtime
user = line_ary[APP_USER_COL]
app = line_ary[APP_APP_COL]
data << {
time: time,
user: user,
app: app
} if (app !~ /^dev/ || SHOW_DEV) && (app !~ /^usr/ || SHOW_USR)
end
return data
end
def generate_app_table(data, apps)
app_data = {}
apps.each do |app|
app_ary = data.select {|h| h[:app] == app}
app_data[app] = {
users: app_ary.map {|h| h[:user]}.uniq.size,
launches: app_ary.map {|h| h[:sess]}.uniq.size
}
end
app_data = app_data.sort_by {|k, v| v[:launches]}.reverse
#turn app_data into an array, rank is assigned based on sessions, then app launches
table_data = []
table_data << ["Rank", "App", "# Launches", "# Users", "# Launches / User"]
rank = 1
app_data.each do |k, v|
table_data << [rank, k, v[:launches], v[:users], (v[:launches] * 1.0 / v[:users]).round(1)]
rank += 1
end
return table_data
end
def generate_user_table(data, project_file)
projects = CSV.read(project_file, headers: true, header_converters: :symbol) if File.exist?(project_file)
top_user = data.map do |obj|
{
key: obj[:user],
sess: obj[:sess],
app: obj[:app]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
num_sessions = v.map { |h| h[:sess] }.uniq.size
num_launches = v.uniq.size
num_apps_used = v.map { |h| h[:app] }.uniq.size
[k, num_sessions, num_launches, (num_launches * 1.0 / num_sessions).round(1), num_apps_used]
end.sort_by { |v| v[1] } .reverse
ranking = 1
table_data = []
table_data << ["Rank", "User", "Name", "# Sessions\\*", "# App Launches", "# Launches / Session", "# Apps Used"]
table_data.last.insert(3, "Institution") if projects #insert at 3, ranking will shift all values
top_user.each do |v|
passwd = Etc.getpwnam(v.first) rescue nil
full_name = passwd ? passwd.gecos : "Unknown"
v.insert(1, full_name)
if projects
group = passwd && Etc.getgrgid(passwd.gid) rescue nil
primary_group = group ? group.name : "Unknown"
institution = if row = projects.find { |row| row[:project] == primary_group }
row[:institution_name]
else
primary_group
end
v.insert(2, institution)
v.insert(0, ranking)
ranking += 1
end
table_data << v
end
return table_data
end
end
lib/ClientData.rb 0 → 100644
+ 157
− 0
View file @ 44ea926d
#!/bin/env ruby
#
# Set options here
#
SHOW_WEB_VER = ENV["SHOW_WEB_VER"] != nil
SHOW_OS_VER = ENV["SHOW_OS_VER"] != nil
CLNT_TIME_COL = 0
CLNT_USER_COL = 1
CLNT_BRWS_COL = 2
CLNT_BRWSV_COL = 3
CLNT_OS_COL = 4
CLNT_OSV_COL = 5
#
# Set dependencies here
#
require 'time'
require "#{LIB}/DataMethods"
class ClientData < DataMethods
def initialize(client_file)
@data = generate_raw_client_data(client_file)
generate_user_session(@data)
@browser_table = generate_browser_table(@data)
@os_table = generate_os_table(@data)
end
attr_reader :browser_table, :os_table
def format_browser_table(last_month_client_data)
table_data = []
table_data << @browser_table.shift
@browser_table.each do |this_month_browser|
last_month_browser = last_month_client_data.return_browser_usage(this_month_browser[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_browser != "N/A")
rank_change = return_rank_change (last_month_browser[0] - this_month_browser[0])
session_change = return_percent_change(this_month_browser[2], last_month_browser[2])
users_change = return_percent_change(this_month_browser[3], last_month_browser[3])
end
table_data << ["#{this_month_browser[0]} #{rank_change}",
"#{this_month_browser[1]}",
"#{this_month_browser[2]} #{session_change}",
"#{this_month_browser[3]} #{users_change}",
]
end
return table_data.first(NUM_WEB + 1) #because of the header
end
def format_os_table(last_month_client_data)
table_data = []
table_data << @os_table.shift
@os_table.each do |this_month_os|
last_month_os = last_month_client_data.return_os_usage(this_month_os[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_os != "N/A")
rank_change = return_rank_change (last_month_os[0] - this_month_os[0])
session_change = return_percent_change(this_month_os[2], last_month_os[2])
users_change = return_percent_change(this_month_os[3], last_month_os[3])
end
table_data << ["#{this_month_os[0]} #{rank_change}",
"#{this_month_os[1]}",
"#{this_month_os[2]} #{session_change}",
"#{this_month_os[3]} #{users_change}",
]
end
return table_data.first(NUM_OS + 1) #because of the header
end
def return_browser_usage(client_name)
@browser_table.each do |browser|
if (browser[1] == client_name)
return browser
end
end
return "N/A"
end
def return_os_usage(os_name)
@os_table.each do |os|
if (os[1] == os_name)
return os
end
end
return "N/A"
end
private
def generate_raw_client_data(client_file)
data = []
File.foreach(client_file) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[CLNT_USER_COL],
time: Time.iso8601(line_ary[CLNT_TIME_COL]).localtime,
browser: line_ary[CLNT_BRWS_COL],
browser_v: line_ary[CLNT_BRWSV_COL],
os: line_ary[CLNT_OS_COL],
os_v: line_ary[CLNT_OSV_COL]
}
end
return data
end
def generate_browser_table(data)
top_browser = data.map do |obj|
{
key: SHOW_WEB_VER ? "#{obj[:browser]} #{obj[:browser_v].to_i}" : obj[:browser],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_browser.map do |v|
v.unshift(rank)
rank += 1
end
top_browser.unshift(["Rank", "Browser", "# Sessions\\*", "# Users"])
return top_browser
end
def generate_os_table(data)
top_os = data.map do |obj|
{
key: SHOW_OS_VER ? "#{obj[:os]} #{obj[:os_v]}" : obj[:os],
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_os.map do |v|
v.unshift(rank)
rank += 1
end
top_os.unshift(["Rank", "OS", "# Sessions\\*", "# Users"])
return top_os
end
end
lib/DataMethods.rb 0 → 100644
+ 45
− 0
View file @ 44ea926d
#!/bin/env ruby
require 'securerandom'
#The main namespace for the data methods
class DataMethods
def return_percent_change(this_month, last_month)
value = (((this_month.to_f / last_month) -1) * 100).round(1)
returnVal = "0.0%"
if (value > 0)
returnVal = "(\u2191#{value}%)"
else
returnVal = "(\u2193#{-value}%)"
end
return returnVal
end
private
def generate_user_session(data)
usr_session = {}
data.sort_by { |h| h[:time] }.each do |h|
user = h[:user]
time = h[:time]
if ( sess_info = usr_session[user] ) && ( (time - sess_info[:start_time]) < (SESSION_TIME * 3600) )
sess = sess_info[:sess]
else
sess = SecureRandom.uuid
usr_session[user] = {
sess: sess,
start_time: time
}
end
h[:sess] = sess
end
end
def return_rank_change(value)
returnVal = "(-)"
if (value > 0)
returnVal = "(\u2191#{value})"
elsif (value < 0)
returnVal = "(\u2193#{-value})"
end
return returnVal
end
end
lib/LocationData.rb 0 → 100644
+ 98
− 0
View file @ 44ea926d
#!/bin/env ruby
#
# Set options here
#
LOC_TIME_COL = 0
LOC_USER_COL = 1
LOC_CITY_COL = 2
LOC_STATE_COL = 3
LOC_CNTRY_COL = 4
#
# Set dependencies here
#
require 'time'
require "#{LIB}/DataMethods"
class LocationData < DataMethods
def initialize(location_file)
@data = generate_raw_location_data(location_file)
generate_user_session(@data)
@location_table = generate_location_table(@data)
end
attr_reader :location_table
def format_location_table(last_month_location_data)
table_data = []
table_data << @location_table.shift
@location_table.each do |this_month_location|
last_month_location = last_month_location_data.return_location_usage(this_month_location[1])
rank_change = "(-)"
session_change = "(-)"
users_change = "(-)"
if (last_month_location != "N/A")
rank_change = return_rank_change (last_month_location[0] - this_month_location[0])
session_change = return_percent_change(this_month_location[2], last_month_location[2])
users_change = return_percent_change(this_month_location[3], last_month_location[3])
end
table_data << ["#{this_month_location[0]} #{rank_change}",
"#{this_month_location[1]}",
"#{this_month_location[2]} #{session_change}",
"#{this_month_location[3]} #{users_change}",
]
end
return table_data.first(NUM_CITY + 1) #because of the header
end
def return_location_usage(location_name)
@location_table.each do |location|
if (location[1] == location_name)
return location
end
end
return "N/A"
end
private
def generate_raw_location_data(location_file)
data = []
File.foreach(location_file) do |line|
line_ary = line.split(DELIM)
data << {
user: line_ary[LOC_USER_COL],
time: Time.iso8601(line_ary[LOC_TIME_COL]).localtime,
city: line_ary[LOC_CITY_COL],
state: line_ary[LOC_STATE_COL],
country: line_ary[LOC_CNTRY_COL],
}
end
return data #array
end
def generate_location_table(data)
top_location = data.map do |obj|
{
key: "#{obj[:city]}, #{obj[:state]}, #{obj[:country]}",
sess: obj[:sess],
user: obj[:user]
}
end.uniq.group_by { |obj| obj[:key] }.map do |k, v|
[k, v.size, v.map { |h| h[:user] }.uniq.size ]
end.sort_by { |v| v[1] } .reverse
rank = 1
top_location.map do |v|
v.unshift(rank)
rank += 1
end
top_location.unshift(["Rank", "City", "# Sessions\\*", "# Users"])
return top_location #array
end
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment