From 185efaf82ebe980ce2a3f1eed96803c5813ad7f9 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 2 Oct 2016 11:05:56 -0700 Subject: [PATCH 1/4] Strip out count dictionary and remove incremental operations. --- advanced/Scripts/dnsmasq_parse.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/advanced/Scripts/dnsmasq_parse.py b/advanced/Scripts/dnsmasq_parse.py index 24428071..01a23e28 100644 --- a/advanced/Scripts/dnsmasq_parse.py +++ b/advanced/Scripts/dnsmasq_parse.py @@ -106,21 +106,18 @@ def convert_date(ds): def parse_query(query): m = q_re.match(query) if m is not None: - counts['qc'] += 1 add_query(m.group(4), m.group(2), m.group(3), m.group(1)) def parse_forward(query): m = f_re.match(query) if m is not None: - counts['fc'] += 1 add_forward(m.group(3), m.group(2), m.group(1)) def parse_reply(query): m = r_re.match(query) if m is not None: - counts['rc'] += 1 add_reply(m.group(4), m.group(2), m.group(3), m.group(1)) @@ -148,8 +145,6 @@ if len(sys.argv) != 2: logfile = sys.argv[1] -counts = {'lc': 0, 'qc': 0, 'fc': 0, 'rc': 0, 'bc':0} - # Create the SQLite connection conn = sqlite3.connect('/etc/pihole/pihole.db') c = conn.cursor() @@ -159,10 +154,6 @@ create_tables() # Parse the log file. for line in open(logfile): line = line.rstrip() - counts['lc'] += 1 - - if (counts['lc'] % 10000) == 0: - conn.commit() if ': query[' in line: parse_query(line) @@ -173,7 +164,4 @@ for line in open(logfile): elif (': reply ' in line) or (': cached ' in line): parse_reply(line) - else: - counts['bc'] += 1 - conn.commit() From 14424939a9e106778b5bbce051c821d80544323e Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 2 Oct 2016 11:26:40 -0700 Subject: [PATCH 2/4] Set correct db in gravity_parse. Start using `with` in dnsmasq --- advanced/Scripts/dnsmasq_parse.py | 25 +++++++++++++------------ advanced/Scripts/gravity_parse.py | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/dnsmasq_parse.py b/advanced/Scripts/dnsmasq_parse.py index 01a23e28..2d0968b6 100644 --- a/advanced/Scripts/dnsmasq_parse.py +++ b/advanced/Scripts/dnsmasq_parse.py @@ -147,21 +147,22 @@ logfile = sys.argv[1] # Create the SQLite connection conn = sqlite3.connect('/etc/pihole/pihole.db') -c = conn.cursor() -create_tables() +with conn: + c = conn.cursor() -# Parse the log file. -for line in open(logfile): - line = line.rstrip() + create_tables() - if ': query[' in line: - parse_query(line) + # Parse the log file. + for line in open(logfile): + line = line.rstrip() - elif ': forwarded ' in line: - parse_forward(line) + if ': query[' in line: + parse_query(line) - elif (': reply ' in line) or (': cached ' in line): - parse_reply(line) + elif ': forwarded ' in line: + parse_forward(line) + + elif (': reply ' in line) or (': cached ' in line): + parse_reply(line) -conn.commit() diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py index 4ac51563..40330c8e 100644 --- a/advanced/Scripts/gravity_parse.py +++ b/advanced/Scripts/gravity_parse.py @@ -18,7 +18,7 @@ import sqlite3 logfile = '/etc/pihole/pihole.2.eventHorizon.txt' # Create the SQLite connection -conn = sqlite3.connect('/etc/pihole/gravity.db') +conn = sqlite3.connect('/etc/pihole/pihole.db') # Python auto-handle commits, no need to call for commits manually with conn: From 64b91c8b971034a3ad0c3b2958249eaddaa36d58 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 2 Oct 2016 11:31:12 -0700 Subject: [PATCH 3/4] `with` open logfile for Python autohandling of closure. --- advanced/Scripts/dnsmasq_parse.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/dnsmasq_parse.py b/advanced/Scripts/dnsmasq_parse.py index 2d0968b6..6bef879f 100644 --- a/advanced/Scripts/dnsmasq_parse.py +++ b/advanced/Scripts/dnsmasq_parse.py @@ -154,15 +154,16 @@ with conn: create_tables() # Parse the log file. - for line in open(logfile): - line = line.rstrip() + with open(logfile) as f: + for line in f: + line = line.rstrip() - if ': query[' in line: - parse_query(line) + if ': query[' in line: + parse_query(line) - elif ': forwarded ' in line: - parse_forward(line) + elif ': forwarded ' in line: + parse_forward(line) - elif (': reply ' in line) or (': cached ' in line): - parse_reply(line) + elif (': reply ' in line) or (': cached ' in line): + parse_reply(line) From 9f29bef17347d27086a78231feecc51f5f7780ad Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 2 Oct 2016 11:36:00 -0700 Subject: [PATCH 4/4] Remove idx creation, sqlite3 handles this automatically. --- advanced/Scripts/dnsmasq_parse.py | 3 --- advanced/Scripts/gravity_parse.py | 1 - 2 files changed, 4 deletions(-) diff --git a/advanced/Scripts/dnsmasq_parse.py b/advanced/Scripts/dnsmasq_parse.py index 6bef879f..29fd4d62 100644 --- a/advanced/Scripts/dnsmasq_parse.py +++ b/advanced/Scripts/dnsmasq_parse.py @@ -62,7 +62,6 @@ r_re = re.compile(r'(.*) dnsmasq\[\d+\]: (reply|cached) (.*) is (.*)') def create_tables(): qt = ''' CREATE TABLE IF NOT EXISTS queries ( - id integer primary key autoincrement, source text, query_type text, name text, @@ -74,7 +73,6 @@ def create_tables(): ft = ''' CREATE TABLE IF NOT EXISTS forwards ( - id integer primary key autoincrement, resolver text, name text, ts datetime @@ -85,7 +83,6 @@ def create_tables(): rt = ''' CREATE TABLE IF NOT EXISTS replies ( - id integer primary key autoincrement, ip text, reply_type text, name text, diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py index 40330c8e..0f4e6ecf 100644 --- a/advanced/Scripts/gravity_parse.py +++ b/advanced/Scripts/gravity_parse.py @@ -33,7 +33,6 @@ with conn: # Ready new table for list of domains gt = ''' CREATE TABLE IF NOT EXISTS gravity ( - idx INTEGER PRIMARY KEY ASC, domain text ) '''