Merge pull request #746 from pi-hole/ds-db

DSchaper db updates

advanced/Scripts/dnsmasq_parse.py

@@ -62,7 +62,6 @@ r_re = re.compile(r'(.*) dnsmasq\[\d+\]: (reply|cached) (.*) is (.*)')
 def create_tables():
     qt = '''
     CREATE TABLE IF NOT EXISTS queries (
-        id integer primary key autoincrement,
         source text,
         query_type text,
         name text,
@@ -74,7 +73,6 @@ def create_tables():
 
     ft = '''
     CREATE TABLE IF NOT EXISTS forwards (
-        id integer primary key autoincrement,
         resolver text,
         name text,
         ts datetime
@@ -85,7 +83,6 @@ def create_tables():
 
     rt = '''
     CREATE TABLE IF NOT EXISTS replies (
-        id integer primary key autoincrement,
         ip text,
         reply_type text,
         name text,
@@ -106,21 +103,18 @@ def convert_date(ds):
 def parse_query(query):
     m = q_re.match(query)
     if m is not None:
-        counts['qc'] += 1
         add_query(m.group(4), m.group(2), m.group(3), m.group(1))
 
 
 def parse_forward(query):
     m = f_re.match(query)
     if m is not None:
-        counts['fc'] += 1
         add_forward(m.group(3), m.group(2), m.group(1))
 
 
 def parse_reply(query):
     m = r_re.match(query)
     if m is not None:
-        counts['rc'] += 1
         add_reply(m.group(4), m.group(2), m.group(3), m.group(1))
 
 
@@ -148,21 +142,18 @@ if len(sys.argv) != 2:
 
 logfile = sys.argv[1]
 
-counts = {'lc': 0, 'qc': 0, 'fc': 0, 'rc': 0, 'bc':0}
-
 # Create the SQLite connection
 conn = sqlite3.connect('/etc/pihole/pihole.db')
-c = conn.cursor()
 
-create_tables()
+with conn:
+    c = conn.cursor()
 
-# Parse the log file.
+    create_tables()
-for line in open(logfile):
+
+    # Parse the log file.
+    with open(logfile) as f:
+        for line in f:
             line = line.rstrip()
-    counts['lc'] += 1
-
-    if (counts['lc'] % 10000) == 0:
-        conn.commit()
 
             if ': query[' in line:
                 parse_query(line)
@@ -173,7 +164,3 @@ for line in open(logfile):
             elif (': reply ' in line) or (': cached ' in line):
                 parse_reply(line)
 
-    else:
-        counts['bc'] += 1
-
-conn.commit()

advanced/Scripts/gravity_parse.py

@@ -33,7 +33,6 @@ with conn:
     # Ready new table for list of domains
     gt = '''
     CREATE TABLE IF NOT EXISTS gravity (
-        idx INTEGER PRIMARY KEY ASC,
         domain text
     )
     '''