From 739027a27d61247d05585905956867db6f14cd44 Mon Sep 17 00:00:00 2001 From: jacobsalmela Date: Wed, 20 Jan 2016 19:40:12 -0600 Subject: [PATCH] scripts to long-term stats These scripts will be necessary for parsing pihole.log before it is flushed and then saving the information into a DB. This information can then be displayed in the Web GUI. mysql-server and php5-mysql will need to be installed. you can then setup the database with this command: mysql -u root -p"raspberry" < /usr/local/bin/setupPiholeStats.sql Once that is done, run sudo php /usr/local/bin/lrs.php --- advanced/Scripts/lrs.php | 140 ++++++++++++++++++++++++++ advanced/Scripts/setupPiholeStats.sql | 22 ++++ 2 files changed, 162 insertions(+) create mode 100644 advanced/Scripts/lrs.php create mode 100644 advanced/Scripts/setupPiholeStats.sql diff --git a/advanced/Scripts/lrs.php b/advanced/Scripts/lrs.php new file mode 100644 index 00000000..a51689de --- /dev/null +++ b/advanced/Scripts/lrs.php @@ -0,0 +1,140 @@ + $value) { + $sql_replace_top_chart_stats = "REPLACE INTO top_chart_stats VALUES ('$pie_chart_date', '$top_id', $value)"; + if (!mysqli_query($db_conn, $sql_replace_top_chart_stats)) { + errorlog("SQL query failed for some reason: $sql_replace_top_chart_stats"); + }else{ + debuglog("SQL query successfully executed: $sql_replace_top_chart_stats"); + } + } +} + +function update_pie_chart_stats($db_conn, $pie_chart_date, $pie_chart_count_query, $pie_chart_count_adver) { + //daily insert + $sql_insert_pie_chart_stats = "INSERT IGNORE INTO pie_chart_stats VALUES ('$pie_chart_date', $pie_chart_count_query, $pie_chart_count_adver)"; + if (!mysqli_query($db_conn, $sql_insert_pie_chart_stats)) { + errorlog("SQL query failed for some reason: $sql_insert_pie_chart_stats"); + }else{ + debuglog("SQL query successfully executed: $sql_insert_pie_chart_stats"); + } + //total calc + $sql_select_pie_chart_stats = "SELECT SUM(query_cnt) AS q_cnt, SUM(adver_cnt) AS a_cnt FROM pie_chart_stats WHERE insert_date != '0000-00-00'"; + if ($res_select_pie_chart_stats = mysqli_query($db_conn, $sql_select_pie_chart_stats)) { + $arr_select_pie_chart_stats = mysqli_fetch_array($res_select_pie_chart_stats); + $total_query = $arr_select_pie_chart_stats['q_cnt']; + $total_adver = $arr_select_pie_chart_stats['a_cnt']; + }else{ + errorlog("SQL query failed for some reason: $sql_select_pie_chart_stats"); + } + //total update + $sql_update_pie_chart_stats = "UPDATE pie_chart_stats SET query_cnt = $total_query, adver_cnt = $total_adver WHERE insert_date = '0000-00-00'"; + if (!mysqli_query($db_conn, $sql_update_pie_chart_stats)) { + errorlog("SQL query failed for some reason: $sql_update_pie_chart_stats"); + }else{ + debuglog("SQL query successfully executed: $sql_update_pie_chart_stats"); + } +} + +//--------------------------------------------------------------------------------------------------- +//main + +debuglog("Script is started"); +$file_name = (isset($argv[1])) ? $argv[1] : $file_name; +if (!file_exists($file_name)) { + errorlog("Provided log file does not exist ".$file_name); +}else{ + debuglog("Trying to read and parse log file ".$file_name); +} + +$pie_chart_date = date("Y-m-d"); +$pie_chart_count_query = 0; +$pie_chart_count_adver = 0; +debuglog("Current date == ".$pie_chart_date); +debuglog("Initial amount of 'query' regexp for ".$pie_chart_date." == ".$pie_chart_count_query); +debuglog("Initial amount of 'adver' regexp for ".$pie_chart_date." == ".$pie_chart_count_adver); +$top_chart_array = array(); + +$file_open = fopen($file_name, "r"); +while (!feof($file_open)) { + $file_line = fgets($file_open); + if (strpos($file_line, $pie_chart_regexp_query)) debuglog("\tIncremented amount of 'query' regexp for ".$pie_chart_date." == ".++$pie_chart_count_query); + if (strpos($file_line, $pie_chart_regexp_adver)) { + debuglog("\tIncremented amount of 'adver' regexp for ".$pie_chart_date." == ".++$pie_chart_count_adver); + array_push($top_chart_array, parse_top_chart_advertiser($file_line)); + } +} +fclose($file_open); + +debuglog("Total amount of 'query' regexp for ".$pie_chart_date." == ".$pie_chart_count_query); +debuglog("Total amount of 'adver' regexp for ".$pie_chart_date." == ".$pie_chart_count_adver); + +//--------------------------------------------------------------------------------------------------- +//mysql + +debuglog("** Trying to connect to database with db_host = $db_host"); +debuglog("** Trying to connect to database with db_user = $db_user"); +debuglog("** Trying to connect to database with db_pass = __skipped"); +debuglog("** Trying to connect to database with db_name = $db_name"); +if ($db_conn = mysqli_connect($db_host, $db_user, $db_pass, $db_name)) { + debuglog("Database connection succesfully established"); +}else{ + errorlog("Could not connect to database, check configured DB credentials"); +} + +debuglog("--- Going to populate data into pie_chart_stats"); +update_pie_chart_stats($db_conn, $pie_chart_date, $pie_chart_count_query, $pie_chart_count_adver); +debuglog("--- Data populated into pie_chart_stats"); +debuglog("--- Going to populate data into top_chart_stats"); +update_top_chart_stats($db_conn, $top_chart_array, $pie_chart_date); +debuglog("--- Data populated into top_chart_stats"); +debuglog("Script is finished"); diff --git a/advanced/Scripts/setupPiholeStats.sql b/advanced/Scripts/setupPiholeStats.sql new file mode 100644 index 00000000..062f58b2 --- /dev/null +++ b/advanced/Scripts/setupPiholeStats.sql @@ -0,0 +1,22 @@ + +-- instal with this command: mysql -u root -p"raspberry" < /usr/local/bin/setupPiholeStats.sql +CREATE DATABASE charts; +USE charts; +DROP TABLE IF EXISTS `pie_chart_stats`; +CREATE TABLE `pie_chart_stats` ( + `insert_date` DATE NOT NULL COMMENT 'date when values were captured', + `query_cnt` INT(10) UNSIGNED NOT NULL COMMENT 'query count appreaprence', + `adver_cnt` INT(10) UNSIGNED NOT NULL COMMENT 'advertiser count apperance', + PRIMARY KEY (`insert_date`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT 'table to store stats for pie (queries vs advertisers) chart'; +-- this row would be updated to store SUM values +-- needs to be added within a table creation +INSERT INTO `pie_chart_stats` VALUES ('0000-00-00', 0, 0); + +DROP TABLE IF EXISTS `top_chart_stats`; +CREATE TABLE `top_chart_stats` ( + `insert_date` DATE NOT NULL COMMENT 'date when values were captured', + `adver_name` VARCHAR (60) NOT NULL COMMENT 'advertiser domain name', + `cnt` MEDIUMINT(7) NOT NULL COMMENT 'advertiser count apperance', + PRIMARY KEY (`insert_date`, `adver_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8;