Skip to content
Snippets Groups Projects
Commit 75b817f2 authored by Matthew Hu's avatar Matthew Hu
Browse files

Simplified searching and result generation with a Splunk API class 

Converted bash script to ruby
TODO:
add functionality to generate monthly data for single host
access constants/vars via command line
generate normal report.md(s) for each app, besides comparison report
parent 7e86be80
No related branches found
No related tags found
1 merge request!3Bash to ruby
Hide whitespace changes
Inline Side-by-side
Showing
with 357 additions and 577 deletions
bin/get_app_data
+ 40
− 101
View file @ 75b817f2
......@@ -3,34 +3,29 @@
#
# Set dependencies here
#
require 'net/http'
require 'uri'
$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
require 'SplunkResult'
require 'csv'
require 'openssl'
require 'rexml/document'
include REXML
require 'uri'
require 'rubygems'
require 'bundler/setup'
#
# Set options here
#
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29"
END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{APP_DIR}/data.app.txt"
COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089'
#Client params
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
URL = "splunk.osc.edu"
SID = ARGV[0]
#Search & output params
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
OFFSET = 10000
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
OUTPUT = "#{APP_DIR}/data.app.txt"
#Constants
DELIM = "|"
TIME_COLUMN = 0
USR_COLUMN = 1 # ga:dimension1
APP_COLUMN = 2 # ga:pagePath
......@@ -91,10 +86,10 @@ APP_TOKEN = {
%r{^/dev/(?<app_name>[^/]+)} => "dev/%{user}/%{app_name}"
}
STDERR.puts "APP_DATA_FILE = #{OUTPUT}"
STDERR.puts "HOST = #{HOST}"
STDERR.puts "START_DATE = #{START_DATE}"
STDERR.puts "END_DATE = #{END_DATE}"
STDERR.puts "APP_DATA_FILE = #{OUTPUT}"
#STDERR.puts "START_DATE = #{START_DATE}"
#STDERR.puts "END_DATE = #{END_DATE}"
class InvalidToken < StandardError; end
......@@ -112,86 +107,30 @@ def parse_token(str, user: "")
raise InvalidToken, "failed to parse app token: #{str}"
end
#polling for done, returns number of events
def splunk_search_state(sid)
#check to see if it is done
done_uri = URI.parse(BASEURL + '/services/search/jobs/' + sid )
done_http = Net::HTTP.new(done_uri.host, done_uri.port)
done_http.use_ssl = true
done_http.verify_mode = OpenSSL::SSL::VERIFY_NONE
done_req = Net::HTTP::Get.new(done_uri.request_uri)
done_req.basic_auth(USER, PASS)
checkVal = 'parsing'
counter = 0
result_count = 0
while (checkVal != 'DONE')
done_res = done_http.request(done_req)
xmldoc = Document.new(done_res.body)
checkVal = XPath.first(xmldoc, "//s:key[@name='dispatchState']").text
progress = XPath.first(xmldoc, "//s:key[@name='doneProgress']").text
result_count = XPath.first(xmldoc, "//s:key[@name='eventCount']").text
if (counter % 5 == 0)
puts "Searching #{APP} for app data... #{progress}/1.0"
end
counter += 1
sleep(1)
end
puts 'Data generated'
return result_count.to_i
end
#returns sid results in relation to count and offset in CSV format; get request
def splunk_search_results(sid, offset)
uri = URI.parse(BASEURL)
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
req_uri = URI::HTTP.build(path: "/services/search/jobs/#{sid}/results", query: "output_mode=csv&count=#{COUNT}&offset=#{offset}")
req = Net::HTTP::Get.new(req_uri)
req.basic_auth(USER, PASS)
res = http.request(req)
return res.body
CLIENT = SplunkResult.new(USER, PASS, URL, SID, {:use_ssl => true, :port => 8089, :mode => 'csv', :offset => OFFSET})
while !CLIENT.complete?
puts "Searching #{APP} for app data... " + CLIENT.percent_complete
sleep(5)
end
#poll, and parse to generate csv
def gather_data(file)
target = open("#{SID_DIR}/#{APP}.app", "r")
sid = target.read
target.close
puts "SID: #{sid}"
request_num = splunk_search_state(sid)
request_num = request_num / COUNT
target = open(file, "w")
for i in 0..request_num
results = splunk_search_results(sid, i*COUNT)
csv = CSV.new(results)
csv.shift
for row in csv
begin
uri = URI(row[APP_COLUMN])
app = uri.path
row[APP_COLUMN] = parse_uri(app, user: row[USR_COLUMN])
row << 1
row << app
target.write "#{row.join(DELIM)}\n"
rescue
$stderr.puts "Warning: #{row}"
end
result_index = CLIENT.num_events/OFFSET
target = open(OUTPUT, "w")
for i in 0..result_index
result = CLIENT.single_result(OFFSET*i)
csv = CSV.new(result)
csv.shift #remove headers
for row in csv
begin
uri = URI(row[APP_COLUMN])
app = uri.path
row[APP_COLUMN] = parse_uri(app, user: row[USR_COLUMN])
row << 1
row << app
target.write "#{row.join(DELIM)}\n"
rescue
$stderr.puts "Warning: #{row}"
end
end
puts 'Done!'
ensure
target.close
end
gather_data(OUTPUT)
target.close
puts 'Done!'
bin/get_client_data
+ 39
− 109
View file @ 75b817f2
......@@ -3,34 +3,28 @@
#
# Set dependencies here
#
require 'net/http'
require 'uri'
$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
require 'SplunkResult'
require 'csv'
require 'openssl'
require 'rexml/document'
include REXML
require 'rubygems'
require 'bundler/setup'
#
# Set options here
#
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29"
END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{APP_DIR}/data.client.txt"
COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089'
#Client params
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
URL = "splunk.osc.edu"
SID = ARGV[0]
#Search & output params
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
OFFSET = 10000
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
OUTPUT = "#{APP_DIR}/data.client.txt"
#Constants
DELIM = "|"
BROWSER_VERSION= 3
BROWSER_MINOR = 6
BROWSER_PATCH = 7
......@@ -38,101 +32,37 @@ OS_VERSION = 5
OS_MINOR = 8
OS_PATCH = 9
STDERR.puts "CLIENT_DATA_FILE = #{OUTPUT}"
STDERR.puts "HOST = #{HOST}"
STDERR.puts "START_DATE = #{START_DATE}"
STDERR.puts "END_DATE = #{END_DATE}"
STDERR.puts "CLIENT_DATA_FILE = #{OUTPUT}"
#STDERR.puts "START_DATE = #{START_DATE}"
#STDERR.puts "END_DATE = #{END_DATE}"
class InvalidToken < StandardError; end
#polling for done, returns number of events
def splunk_search_state(sid)
done_uri = URI.parse(BASEURL + '/services/search/jobs/' + sid )
done_http = Net::HTTP.new(done_uri.host, done_uri.port)
done_http.use_ssl = true
done_http.verify_mode = OpenSSL::SSL::VERIFY_NONE
done_req = Net::HTTP::Get.new(done_uri.request_uri)
done_req.basic_auth(USER, PASS)
checkVal = 'parsing'
counter = 0
result_count = 0
while (checkVal != 'DONE')
done_res = done_http.request(done_req)
xmldoc = Document.new(done_res.body)
checkVal = XPath.first(xmldoc, "//s:key[@name='dispatchState']").text
progress = XPath.first(xmldoc, "//s:key[@name='doneProgress']").text
result_count = XPath.first(xmldoc, "//s:key[@name='eventCount']").text
if (counter % 5 == 0)
puts "Searching #{APP} for client data... #{progress}/1.0"
end
counter += 1
sleep(1)
end
puts 'Data generated'
return result_count.to_i
end
#returns sid results in relation to count and offset in CSV format; get request
def splunk_search_results(sid, offset)
uri = URI.parse(BASEURL)
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
req_uri = URI::HTTP.build(path: "/services/search/jobs/#{sid}/results", query: "output_mode=csv&count=#{COUNT}&offset=#{offset}")
req = Net::HTTP::Get.new(req_uri)
req.basic_auth(USER, PASS)
res = http.request(req)
return res.body
CLIENT = SplunkResult.new(USER, PASS, URL, SID, {:use_ssl => true, :port => 8089, :mode => 'csv', :offset => OFFSET})
while !CLIENT.complete?
puts "Searching #{APP} for app data... " + CLIENT.percent_complete
sleep(5)
end
#poll, and parse to generate csv
def gather_data(file)
target = open("#{SID_DIR}/#{APP}.client", "r")
sid = target.read
target.close
puts "SID: #{sid}"
request_num = splunk_search_state(sid)
request_num = request_num / COUNT
target = open(file, "w")
for i in 0..request_num
results = splunk_search_results(sid, i*COUNT)
csv = CSV.new(results)
csv.shift
for row in csv
if (row[BROWSER_MINOR] != 'unknown' && row[BROWSER_MINOR] != nil)
row[BROWSER_VERSION] = row[BROWSER_VERSION] + '.' + row[BROWSER_MINOR]
if (row[BROWSER_PATCH]!= 'unknown' && row[BROWSER_PATCH] != nil)
row[BROWSER_VERSION] = row[BROWSER_VERSION] + '.' + row[BROWSER_PATCH]
end
end
if(row[OS_MINOR] != 'unknown' && row[OS_MINOR] !=nil)
row[OS_VERSION] = row[OS_VERSION] + '.' + row[OS_MINOR]
if(row[OS_PATCH] != 'unknown' && row[OS_PATCH] !=nil)
row[OS_VERSION] = row[OS_VERSION] + '.' + row[OS_PATCH]
end
end
row[6] = 1
row.pop
row.pop
row.pop
target.write "#{row.join(DELIM)}\n"
end
events = CLIENT.num_events
target = open(OUTPUT, "w")
for i in 0..(events/OFFSET)
result = CLIENT.single_result(OFFSET*i)
csv = CSV.new(result)
csv.shift #remove headers
for row in csv
#condense browser and OS versions
row[BROWSER_VERSION] = row[BROWSER_VERSION] + '.' + row[BROWSER_MINOR] if (row[BROWSER_MINOR] != 'unknown' && row[BROWSER_MINOR] != nil)
row[BROWSER_VERSION] = row[BROWSER_VERSION] + '.' + row[BROWSER_PATCH] if (row[BROWSER_PATCH]!= 'unknown' && row[BROWSER_PATCH] != nil)
row[OS_VERSION] = row[OS_VERSION] + '.' + row[OS_MINOR] if (row[OS_MINOR] != 'unknown' && row[OS_MINOR] !=nil)
row[OS_VERSION] = row[OS_VERSION] + '.' + row[OS_PATCH] if (row[OS_PATCH] != 'unknown' && row[OS_PATCH] !=nil)
row[6] = 1
row.pop(3)
target.write "#{row.join(DELIM)}\n"
end
puts "Done!"
ensure
target.close
end
gather_data(OUTPUT)
target.close
puts 'Done!'
bin/get_location_data
+ 32
− 99
View file @ 75b817f2
......@@ -3,119 +3,52 @@
#
# Set dependencies here
#
require 'net/http'
require 'uri'
$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
require 'SplunkResult'
require 'csv'
require 'openssl'
require 'rexml/document'
include REXML
require 'rubygems'
require 'bundler/setup'
#
# Set options here
#
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
START_DATE = ENV["START_DATE"] || "2016-11-29"
END_DATE = ENV["END_DATE"] || "today"
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "./"
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
SID_DIR = "#{DIR}/sid"
OUTPUT = "#{APP_DIR}/data.location.txt"
COUNT = 60000
BASEURL = 'https://splunk.osc.edu:8089'
#Client params
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
URL = "splunk.osc.edu"
SID = ARGV[0]
#Search & output params
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
OFFSET = 10000
APP_DIR = ENV["APP_DIR"] || "./output/current/#{APP}"
OUTPUT = "#{APP_DIR}/data.location.txt"
#Constants
DELIM = "|"
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
STDERR.puts "LOCATION_DATA_FILE = #{OUTPUT}"
STDERR.puts "HOST = #{HOST}"
STDERR.puts "START_DATE = #{START_DATE}"
STDERR.puts "END_DATE = #{END_DATE}"
STDERR.puts "LOCATION_DATA_FILE = #{OUTPUT}"
#STDERR.puts "START_DATE = #{START_DATE}"
#STDERR.puts "END_DATE = #{END_DATE}"
class InvalidToken < StandardError; end
#polling for done, returns number of events
def splunk_search_state(sid)
done_uri = URI.parse(BASEURL + '/services/search/jobs/' + sid )
done_http = Net::HTTP.new(done_uri.host, done_uri.port)
done_http.use_ssl = true
done_http.verify_mode = OpenSSL::SSL::VERIFY_NONE
done_req = Net::HTTP::Get.new(done_uri.request_uri)
done_req.basic_auth(USER, PASS)
checkVal = 'parsing'
counter = 0
result_count = 0
while (checkVal != 'DONE')
done_res = done_http.request(done_req)
xmldoc = Document.new(done_res.body)
checkVal = XPath.first(xmldoc, "//s:key[@name='dispatchState']").text
progress = XPath.first(xmldoc, "//s:key[@name='doneProgress']").text
result_count = XPath.first(xmldoc, "//s:key[@name='eventCount']").text
if (counter % 5 == 0)
puts "Searching #{APP} for location data... #{progress}/1.0"
end
counter += 1
sleep(1)
end
puts 'Data generated'
return result_count.to_i
end
#returns sid results in relation to count and offset in CSV format; get request
def splunk_search_results(sid, offset)
uri = URI.parse(BASEURL)
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
req_uri = URI::HTTP.build(path: "/services/search/jobs/#{sid}/results", query: "output_mode=csv&count=#{COUNT}&offset=#{offset}")
req = Net::HTTP::Get.new(req_uri)
req.basic_auth(USER, PASS)
res = http.request(req)
return res.body
CLIENT = SplunkResult.new(USER, PASS, URL, SID, {:use_ssl => true, :port => 8089, :mode => 'csv', :offset => OFFSET})
while !CLIENT.complete?
puts "Searching #{APP} for app data... " + CLIENT.percent_complete
sleep(5)
end
#poll, and parse to generate csv
def gather_data(file)
target = open("#{SID_DIR}/#{APP}.location", "r")
sid = target.read
target.close
puts "SID: #{sid}"
request_num = splunk_search_state(sid)
request_num = request_num / COUNT
target = open(file, "w")
for i in 0..request_num
results = splunk_search_results(sid, i*COUNT)
csv = CSV.new(results)
csv.shift
for row in csv
row << 1
target.write "#{row.join(DELIM)}\n"
end
events = CLIENT.num_events
target = open(OUTPUT, "w")
for i in 0..(events/OFFSET)
result = CLIENT.single_result(OFFSET*i)
csv = CSV.new(result)
csv.shift #remove headers
for row in csv
row << 1
target.write "#{row.join(DELIM)}\n"
end
puts "Done!"
ensure
target.close
end
gather_data(OUTPUT)
target.close
puts 'Done!'
bin/get_monthly_data deleted 100755 → 0
+ 0
− 30
View file @ 7e86be80
#!/bin/env ruby
#
# Do not modify below
#
require 'rubygems'
require 'bundler/setup'
require 'date'
START_DATE = Date.strptime(ENV["START_DATE"] || "2017-01", "%Y-%m")
END_DATE = Date.strptime(ENV["END_DATE"] || "2018-03", "%Y-%m")
REPORT_FILE = Pathname.new(ENV["REPORT_FILE"] || "monthly_report.md")
(START_DATE..END_DATE).map do |d|
[
Date.new(d.year, d.month, 1),
Date.new(d.year, d.month, -1)
]
end.uniq.each do |first, last|
set_show_usr_env = ENV['SHOW_USR'] ? 'SHOW_USR=' + ENV['SHOW_USR']+ ' ' : ''
set_host_env = ENV['HOST'] ? 'HOST=' + ENV['HOST']+ ' ' : ''
puts "# Execute this command to get data for #{first} .. #{last}"
puts ""
puts %{#{set_show_usr_env}#{set_host_env}START_DATE="#{first}" END_DATE="#{last}" bin/get_data && #{set_show_usr_env}bin/metrics >> #{REPORT_FILE}}
puts %{echo "" >> #{REPORT_FILE}}
puts ""
end
bin/metrics
+ 13
− 16
View file @ 75b817f2
......@@ -3,24 +3,21 @@
#
# Set options here
#
APP = ENV["APP"] || "ondemand"
HOST = ENV["HOST"] || "ondemand.osc.edu"
DIR = ENV["DIR"] || "./"
OUTPUT_DIR = ENV["OUTPUT_DIR"] || "#{DIR}/output"
PROJ_FILE = "#{DIR}/dbreport.txt"
APP_FILE = "#{OUTPUT_DIR}/current/#{APP}/data.app.txt"
CLIENT_FILE = "#{OUTPUT_DIR}/current/#{APP}/data.client.txt"
LOCATION_FILE = "#{OUTPUT_DIR}/current/#{APP}/data.location.txt"
LAST_APP_FILE = "#{OUTPUT_DIR}/previous/#{APP}/data.app.txt"
LAST_CLIENT_FILE = "#{OUTPUT_DIR}/previous/#{APP}/data.client.txt"
LAST_LOCATION_FILE = "#{OUTPUT_DIR}/previous/#{APP}/data.location.txt"
APP = ENV["APP"] || "ondemand"
CURRENT_OUTPUT = ENV["CURRENT_OUTPUT"]
COMPARE_OUTPUT = ENV["COMPARE_OUTPUT"]
PROJ_FILE = ENV["PROJ_FILE"]
CURRENT_APP = "#{CURRENT_OUTPUT}#{APP}/data.app.txt"
CURRENT_CLIENT = "#{CURRENT_OUTPUT}#{APP}/data.client.txt"
CURRENT_LOCATION = "#{CURRENT_OUTPUT}#{APP}/data.location.txt"
COMPARE_APP = "#{COMPARE_OUTPUT}#{APP}/data.app.txt"
COMPARE_CLIENT = "#{COMPARE_OUTPUT}#{APP}/data.client.txt"
COMPARE_LOCATION = "#{COMPARE_OUTPUT}#{APP}/data.location.txt"
#
# Set dependencies here
#
$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
require 'AppData'
require 'ClientData'
......@@ -78,8 +75,8 @@ class Report
end
def main
report1 = Report.new(APP_FILE, CLIENT_FILE, LOCATION_FILE, PROJ_FILE)
report2 = Report.new(LAST_APP_FILE, LAST_CLIENT_FILE, LAST_LOCATION_FILE, PROJ_FILE)
report1 = Report.new(CURRENT_APP, CURRENT_CLIENT, CURRENT_LOCATION, PROJ_FILE)
report2 = Report.new(COMPARE_APP, COMPARE_CLIENT, COMPARE_LOCATION, PROJ_FILE)
compare = DataMethods.new
totalUser = compare.return_percent_change(report1.app.user_cnt, report2.app.user_cnt)
totalApp = compare.return_percent_change(report1.app.app_launches, report2.app.app_launches)
......
bin/search_and_report.rb 0 → 100644
+ 110
− 0
View file @ 75b817f2
#!/bin/env ruby
#Libraries
$LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
require 'fileutils'
require 'SplunkSearch'
#Constants and Presets
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
DIR = ENV["DIR"] || "."
# Search parameters
# https://docs.splunk.com/Documentation/Splunk/8.0.4/SearchReference/SearchTimeModifiers
TIME_UNITS = "d"
#START/END times must NOT be positive integers
CURRENT_START = 0
CURRENT_END = -1
COMPARE_START = -1
COMPARE_END = -2
CURRENT_OUTPUT = './output/current/'
COMPARE_OUTPUT = './output/compare/'
REPORT_OUTPUT = './output/reports/'
PROJ_FILE = "./dbreport.txt"
IMAGE = "/users/PZS0714/johrstrom/Public/images/sing/pandoc.sif"
ENV["CURRENT_OUTPUT"] = CURRENT_OUTPUT
ENV["COMPARE_OUTPUT"] = COMPARE_OUTPUT
ENV["REPORT_OUTPUT"] = REPORT_OUTPUT
ENV["PROJ_FILE"] = PROJ_FILE
#Create directories
FileUtils.mkdir_p [CURRENT_OUTPUT, COMPARE_OUTPUT, REPORT_OUTPUT]
#Start Searches
CLIENT = SplunkSearch.new(USER, PASS, "splunk.osc.edu", {:use_ssl => true, :port => 8089})
def splunk_search(app, host, start_time, end_time)
time_params = start_time == 0 ? time_params = "earliest=#{end_time}#{TIME_UNITS}@#{TIME_UNITS} latest=@#{TIME_UNITS}" : "earliest=#{end_time}#{TIME_UNITS}@#{TIME_UNITS} latest=#{start_time}#{TIME_UNITS}@#{TIME_UNITS}"
basic_query = "search source=/var/log/httpd24/#{host}_access_ssl.log #{time_params} | where user!=\"-\" | "
app_query = "#{basic_query} where http_referer!=\"-\" | search *#{host}/pun/ | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+http_referer| dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\")| table mytime, user, http_referer"
client_query = "#{basic_query} lookup user_agents http_user_agent | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+ua_family | dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, ua_family, ua_major, ua_os_family, ua_os_major, ua_minor, ua_patch, ua_os_minor, ua_os_patch"
location_query = "#{basic_query} iplocation src_ip | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+City| dedup session |eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, City, Region, Country"
app_sid = CLIENT.search(app_query)
client_sid = CLIENT.search(client_query)
location_sid = CLIENT.search(location_query)
abort "Error: Search was not queried; please check username/password input" unless !app_sid.empty? && !client_sid.empty? && !location_sid.empty?
return {:app => app_sid, :client => client_sid, :location => location_sid}
end
def get_data(app, host, sids, output_dir)
app_dir = output_dir + app
ENV["APP"] = app
ENV["HOST"] = host
ENV["APP_DIR"] = app_dir
FileUtils.mkdir_p app_dir
STDERR.puts "Getting APP data for #{app}"
abort "App data not available" unless system "bin/get_app_data #{sids[:app]}"
STDERR.puts "Getting CLIENT data for #{app}:"
abort "Client data not available" unless system "bin/get_client_data #{sids[:client]}"
STDERR.puts "Getting LOCATION data for #{app}:"
abort "Location data not available" unless system "bin/get_location_data #{sids[:location]}"
end
#report Generation
def generate_report(app)
ENV["APP"] = app
report = "#{REPORT_OUTPUT}" + "#{app}.md"
docx = "#{REPORT_OUTPUT}" + "#{app}_report.docx"
pandoc_args = "-f markdown -t docx -o #{docx} #{report}"
system "bin/metrics > #{report}"
system "singularity run #{IMAGE} #{pandoc_args}"
end
#current
puts "Searching for CURRENT data"
current_awesim = splunk_search('awesim', 'apps.awesim.org', CURRENT_START, CURRENT_END)
current_totalsim = splunk_search('totalsim', 'apps.totalsim.us', CURRENT_START, CURRENT_END)
current_ondemand = splunk_search('ondemand', 'ondemand.osc.edu', CURRENT_START, CURRENT_END)
current_stats = splunk_search('stats', 'stat.osc.edu', CURRENT_START, CURRENT_END)
puts "CURRENT searches have been sucessfully queried, processing data"
get_data('awesim', 'apps.awesim.org', current_awesim, CURRENT_OUTPUT)
get_data('totalsim', 'apps.totalsim.us', current_totalsim, CURRENT_OUTPUT)
get_data('ondemand', 'ondemand.osc.edu', current_ondemand, CURRENT_OUTPUT)
get_data('stats', 'stat.osc.edu', current_stats, CURRENT_OUTPUT)
puts "CURRENT search data can be found at #{CURRENT_OUTPUT}"
#compare
puts "Searching for COMPARE data"
compare_awesim = splunk_search('awesim', 'apps.awesim.org', COMPARE_START, COMPARE_END)
compare_totalsim = splunk_search('totalsim', 'apps.totalsim.us', COMPARE_START, COMPARE_END)
compare_ondemand = splunk_search('ondemand', 'ondemand.osc.edu', COMPARE_START, COMPARE_END)
compare_stats = splunk_search('stats', 'stat.osc.edu', COMPARE_START, COMPARE_END)
puts "COMPARE searches have been sucessfully queried, processing data"
get_data('awesim', 'apps.awesim.org', compare_awesim, COMPARE_OUTPUT)
get_data('totalsim', 'apps.totalsim.us', compare_totalsim, COMPARE_OUTPUT)
get_data('ondemand', 'ondemand.osc.edu', compare_ondemand, COMPARE_OUTPUT)
get_data('stats', 'stat.osc.edu', compare_stats, COMPARE_OUTPUT)
puts "COMPARE search data can be found at #{COMPARE_OUTPUT}"
#generate reports
puts "Generating reports..."
generate_report('awesim')
generate_report('totalsim')
generate_report('ondemand')
generate_report('stats')
puts "Generated reports can be found at #{REPORT_OUTPUT}"
bin/get_data → bin/startEndDateDeprecated.rb
+ 1
− 12
View file @ 75b817f2
......@@ -25,15 +25,4 @@ def start_end_date(units, diff)
ENV["END_DATE"] = end_date.strftime("%F")
end
start_end_date(TIME_UNITS, TIME_DIFF)
STDERR.puts "should really cd to #{File.expand_path(File.dirname(__FILE__))}"
STDERR.puts "Getting APP data for #{APP}:"
abort "App data not available" unless system "bin/get_app_data"
STDERR.puts "Getting CLIENT data for #{APP}:"
abort "Client data not available" unless system "bin/get_client_data"
STDERR.puts "Getting LOCATION data for #{APP}:"
abort "Location data not available" unless system "bin/get_location_data"
start_end_date(TIME_UNITS, TIME_DIFF)
\ No newline at end of file
bin/start_searches deleted 100755 → 0
+ 0
− 70
View file @ 7e86be80
#!/bin/env ruby
require 'net/http'
require 'uri'
require 'openssl'
require 'rexml/document'
include REXML
HOST = ENV["HOST"] || "ondemand.osc.edu"
APP = ENV["APP"] || "ondemand"
USER = ENV["USER"] || "admin"
PASS = ENV["PASS"] || "pass"
TIME_DIFF = (ENV["TIME_DIFF"] || "1").to_i
TIME_UNITS = ENV["TIME_UNITS"] || "mon"
DIR = ENV["DIR"] || "."
SID_DIR = "#{DIR}/sid"
time_params = "earliest=-1#{TIME_UNITS} latest=now"
if (TIME_DIFF > 1)
time_params = "earliest=-#{TIME_DIFF}#{TIME_UNITS}@#{TIME_UNITS} latest=-#{TIME_DIFF - 1}#{TIME_UNITS}@#{TIME_UNITS}"
elsif (TIME_DIFF <= 0)
abort "Incorrect time difference parameter"
end
BASEURL = 'https://splunk.osc.edu:8089'
BASIC_QUERY = "search source=/var/log/httpd24/#{HOST}_access_ssl.log #{time_params} | where user!=\"-\" | "
APP_QUERY = "#{BASIC_QUERY} where http_referer!=\"-\" | search *#{HOST}/pun/ | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+http_referer| dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\")| table mytime, user, http_referer"
CLIENT_QUERY = "#{BASIC_QUERY} lookup user_agents http_user_agent | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+ua_family | dedup session | eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, ua_family, ua_major, ua_os_family, ua_os_major, ua_minor, ua_patch, ua_os_minor, ua_os_patch"
LOCATION_QUERY = "#{BASIC_QUERY} iplocation src_ip | eval session=strftime(_time, \"\%Y-\%m-\%d\")+user+City| dedup session |eval mytime=strftime(_time, \"\%Y-\%m-\%dT\%H:\%M:\%SZ\") | table mytime, user, City, Region, Country"
#send search request
def splunk_search(search_params, name)
#setup search params for splunk
search_uri = URI.parse(BASEURL + '/services/search/jobs')
search_http = Net::HTTP.new(search_uri.host, search_uri.port)
search_http.use_ssl = true
search_http.verify_mode = OpenSSL::SSL::VERIFY_NONE
#set a request form and data
search_req = Net::HTTP::Post.new(search_uri.request_uri)
search_req.basic_auth(USER, PASS)
search_req.set_form_data({'search'=> search_params})
#send a post request to splunk
search_res = search_http.request(search_req)
if search_res.code.to_i == 401
return search_res.code
end
#parse xml into doc and get searchID
xmldoc = Document.new(search_res.body)
sid = XPath.first(xmldoc, '//sid').text
puts "#{APP} #{name} search ID: " + sid
target = open("#{SID_DIR}/#{APP}.#{name}", "w")
target.write sid
target.close
return search_res.code
end
def main()
var = splunk_search(APP_QUERY, "app")
var = splunk_search(CLIENT_QUERY, "client")
var = splunk_search(LOCATION_QUERY, "location")
abort "Wrong User or Password" unless var.to_i == 201
end
main
generate_docx.sh deleted 100644 → 0
+ 0
− 32
View file @ 7e86be80
#!/bin/sh
function generate_docx(){
APP=$1
APP_DIR=$DIR/output/current/$APP
IN_FILE=$APP_DIR/report.md
OUT_FILE=$DATA_DIR/$APP-$SHORT_DATE.docx
#pandoc
img=/users/PZS0714/johrstrom/Public/images/sing/pandoc.sif
pandoc_args="-f markdown -t docx -o $OUT_FILE $IN_FILE"
singularity run $img $pandoc_args
}
function main(){
DIR=$(dirname $(readlink -f "$0"))
SHORT_DATE=$(date +%Y-%m)
#New directory for docx to generate into
DATA_DIR=$DIR/output/reports
mkdir $DATA_DIR
# Need to unset LD_PRELOAD for pandoc to work
unset LD_PRELOAD
generate_docx 'awesim'
generate_docx 'totalsim'
generate_docx 'ondemand'
generate_docx 'stats'
}
main
generate_reports.sh
+ 11
− 107
View file @ 75b817f2
#!/bin/sh
function start_search(){
export HOST=$1
export APP=$2
$DIR/bin/start_searches
}
function get_data(){
export HOST=$1
export APP=$2
export APP_DIR=$DATA_DIR/$APP
mkdir $APP_DIR
$DIR/bin/get_data
}
function search_month(){
export TIME_DIFF=$1
DATA_DIR=$2
mkdir $DATA_DIR
SID_DIR=$DIR/sid #SID directory
mkdir $SID_DIR
start_search 'apps.awesim.org' 'awesim'
start_search 'apps.totalsim.us' 'totalsim'
start_search 'ondemand.osc.edu' 'ondemand'
start_search 'stat.osc.edu' 'stats'
if [ $? -eq 1 ]; then
rm -rf $SID_DIR
exit
fi
get_data 'apps.awesim.org' 'awesim'
get_data 'apps.totalsim.us' 'totalsim'
get_data 'ondemand.osc.edu' 'ondemand'
get_data 'stat.osc.edu' 'stats'
rm -rf $SID_DIR
}
function generate_monthly_report(){
export HOST=$1
export APP=$2
if [ "$APP" != "ondemand" ]; then
export SHOW_USR=1
fi
$DIR/bin/metrics > $CURRENT/$APP/report.md
#process docx command
IN_FILE=$CURRENT/$APP/report.md
OUT_FILE=$DOCX_DIR/$APP-$SHORT_DATE.docx
#pandoc
img=/users/PZS0714/johrstrom/Public/images/sing/pandoc.sif
pandoc_args="-f markdown -t docx -o $OUT_FILE $IN_FILE"
singularity run $img $pandoc_args
}
function main(){
#get password; user is already preset and can be checked with `whoami`
read -s -p 'Pass: ' pass
echo
export USER=`whoami`
export PASS=$pass
export DIR=$(dirname $(readlink -f "$0"))
module load ruby # load ruby 2.5 on owens login node
unset LD_PRELOAD # Need to unset LD_PRELOAD for pandoc to work
SHORT_DATE=$(date +%Y-%m)
#Options
export OUTPUT_DIR=$DIR/output #change here to change output directories
# https://docs.splunk.com/Documentation/Splunk/8.0.4/SearchReference/SearchTimeModifiers
export TIME_UNITS="d" # splunk time modifiers. Adjust according to link above
COMPARE=2 #input relative time range
# For TIME_UNITS='mon' and COMPARE=2, results will be generated both last month and the month before last month
# So it searches from (2-1) months ago & (1-now) months ago
# Then report(s).md are generated in the current (1-now) directory
# These report(s).md are pushed through pandoc to generate docx under ./output/reports
mkdir $OUTPUT_DIR
CURRENT=$OUTPUT_DIR/current
PREVIOUS=$OUTPUT_DIR/previous
DOCX_DIR=$DIR/output/reports
mkdir $DOCX_DIR
#run searches and get data
search_month $COMPARE $PREVIOUS
search_month '1' $CURRENT
#generate reports
generate_monthly_report 'apps.awesim.org' 'awesim'
generate_monthly_report 'apps.totalsim.us' 'totalsim'
generate_monthly_report 'ondemand.osc.edu' 'ondemand'
generate_monthly_report 'stat.osc.edu' 'stats'
}
main
#Output directory -must be exported for metrics
#Data directory (current and previous)
#App directory -must be exported for ease of use in get____data, fxn doesn't know if current or previous dir
#Output files -handled by lower level
\ No newline at end of file
#Set variables
read -s -p 'Pass: ' pass
echo
export USER=`whoami`
export PASS=$pass
DIR=$(dirname $(readlink -f "$0"))
#
module load ruby
unset LD_PRELOAD
ruby $DIR/bin/generate_reports2.rb
lib/AppData.rb
+ 1
− 1
View file @ 75b817f2
......@@ -187,7 +187,7 @@ class AppData < DataMethods
num_launches = v.uniq.size
num_apps_used = v.map { |h| h[:app] }.uniq.size
[k, num_sessions, num_launches, (num_launches * 1.0 / num_sessions).round(1), num_apps_used]
end.sort_by { |v| v[2] } .reverse.sort_by {|v| v[1]}.reverse
end.sort_by { |v| v[2] } .reverse
#Sort by sessions then app launches
rank = 1
......
lib/SplunkClient.rb 0 → 100644
+ 43
− 0
View file @ 75b817f2
#!/bin/env ruby
require 'net/http'
require 'uri'
require 'openssl'
class SplunkClient
def initialize(username, password, host, opts = {})
@user = username
@pass = password
@host = host
@port = opts[:port] || 8089
@use_ssl = opts[:use_ssl] || false
end
# path - string - /services/path/to/service/wanted
# opts - an optional hash containing request params
def get_request(path, opts = {})
uri = URI::HTTP.build(host: @host, port: @port, path: path, query: URI.encode_www_form(opts))
req = Net::HTTP::Get.new(uri)
req.basic_auth(@user, @pass)
splunk_http_request.request(req).body
end
# path - string - /services/path/to/service/wanted
# data - hash - a hash representing the data
# header - hash - an optional hash containing header requests
def post_request(path, data, header = nil)
req = Net::HTTP::Post.new(path)
req.set_form_data(data)
req.basic_auth(@user, @pass)
splunk_http_request.request(req).body
end
private
def splunk_http_request
request = Net::HTTP.new(@host, @port)
request.use_ssl = @use_ssl
request.verify_mode = OpenSSL::SSL::VERIFY_NONE
request
end
end
\ No newline at end of file
lib/SplunkResult.rb 0 → 100644
+ 55
− 0
View file @ 75b817f2
#!/bin/env ruby
require 'SplunkClient'
require 'nokogiri'
class SplunkResult < SplunkClient
def initialize(username, password, host, sid, opts = {})
#client params
@user = username
@pass = password
@host = host
@port = opts[:port] || 8089
@use_ssl = opts[:use_ssl] || false
#search params
@sid = sid
@output_mode= opts[:mode] || 'csv'
@offset = opts[:offset] || 10000
end
# when called, returns the percentage of the search completed
def percent_complete
xml = get_request("/services/search/jobs/#{@sid}")
doc = Nokogiri::XML(xml)
progress = doc.xpath("//s:key[@name='doneProgress']").text
return "#{progress}/1.0"
end
# returns boolean - if the search is complete
def complete?
xml = get_request("/services/search/jobs/#{@sid}")
doc = Nokogiri::XML(xml)
text = doc.xpath("//s:key[@name='isDone']").text
text.to_i == 1
end
def results
numOfSearches = numEvents / @OFFSET #the number of searches we must make to return all values
result = Array.new
if (complete?)
for i in 0..numOfSearches
result.push( get_request("/services/search/jobs/#{@sid}/results", {'output_mode' => @output_mode, 'count' => @offset, 'offset' => (@offset * i)}))
end
end
return result
end
def single_result(offset)
return get_request("/services/search/jobs/#{@sid}/results", {'output_mode' => @output_mode, 'count' => @offset, 'offset' => offset})
end
def num_events
xml = get_request("/services/search/jobs/#{@sid}")
doc = Nokogiri::XML(xml)
text = doc.xpath("//s:key[@name='eventCount']").text
return text.to_i
end
end
\ No newline at end of file
lib/SplunkSearch.rb 0 → 100644
+ 12
− 0
View file @ 75b817f2
#!/bin/env ruby
#Libraries
require './lib/SplunkClient'
require 'nokogiri'
class SplunkSearch < SplunkClient
def search(query)
xml = post_request('/services/search/jobs', {'search' => query})
doc = Nokogiri::XML(xml)
return doc.xpath('//sid').text
end
end
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment