From 04b5e1a7d354e149b4ea24c01c6101b696cd4506 Mon Sep 17 00:00:00 2001
From: Dan Schaper <dschaper@ganymeade.com>
Date: Sat, 1 Oct 2016 20:03:50 -0700
Subject: [PATCH 1/4] Remove extra DELETE FROM - Table has just been created,
 should alread by empty.

---
 advanced/Scripts/gravity_parse.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py
index 529eb775..011a6bfd 100644
--- a/advanced/Scripts/gravity_parse.py
+++ b/advanced/Scripts/gravity_parse.py
@@ -71,19 +71,16 @@ counts = {'lc': 0}
 
 # Create the SQLite connection
 conn = sqlite3.connect('/etc/pihole/pihole.db')
-conn.text_factory = str  # I don't like it as a fix, but it works for now
-c = conn.cursor()
 
-create_tables()
+with conn:
+    c = conn.cursor()
 
-sql = "DELETE FROM gravity"
-c.execute(sql)
-conn.commit()
+    create_tables()
 
-# Parse the log file.
-for line in open(logfile):
-    line = line.rstrip()
-    counts['lc'] += 1
+    # Parse the log file.
+    for line in open(logfile):
+        line = line.rstrip()
+        counts['lc'] += 1
 
     if (counts['lc'] % 10000) == 0:
         conn.commit()

From 2e331cbd73df1c17ae0940f971d94f74ff3faa70 Mon Sep 17 00:00:00 2001
From: Dan Schaper <dschaper@ganymeade.com>
Date: Sat, 1 Oct 2016 20:12:56 -0700
Subject: [PATCH 2/4] Try WAL mode

---
 advanced/Scripts/gravity_parse.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py
index 011a6bfd..2848da8e 100644
--- a/advanced/Scripts/gravity_parse.py
+++ b/advanced/Scripts/gravity_parse.py
@@ -77,6 +77,9 @@ with conn:
 
     create_tables()
 
+    # enable WAL mode
+    c.execute('PRAGMA journal_mode=WAL;')
+
     # Parse the log file.
     for line in open(logfile):
         line = line.rstrip()
@@ -87,5 +90,3 @@ with conn:
 
     sql = "INSERT INTO gravity (domain) VALUES (?)"
     c.execute(sql, (line,))
-
-conn.commit()

From e6873e0ebdc8e81b83e334d1403324815e46d278 Mon Sep 17 00:00:00 2001
From: Dan Schaper <dschaper@ganymeade.com>
Date: Sat, 1 Oct 2016 21:13:55 -0700
Subject: [PATCH 3/4] Commits squashed, lots of changes.

---
 advanced/Scripts/gravity_parse.py | 47 +++++++++----------------------
 1 file changed, 13 insertions(+), 34 deletions(-)

diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py
index 2848da8e..29d50be6 100644
--- a/advanced/Scripts/gravity_parse.py
+++ b/advanced/Scripts/gravity_parse.py
@@ -43,50 +43,29 @@
 
 import sqlite3
 
-#-----------------------------------------------------------------------------
-# Functions
-#-----------------------------------------------------------------------------
-def create_tables():
-
-    qt = 'DROP TABLE IF EXISTS gravity'
-    c.execute(qt)
-    conn.commit()
-
-    qt = '''
-    CREATE TABLE IF NOT EXISTS gravity (
-        idx integer primary key autoincrement,
-        domain text
-    )
-    '''
-    c.execute(qt)
-    conn.commit()
-
-#-----------------------------------------------------------------------------
-# Main
-#-----------------------------------------------------------------------------
-
 logfile = '/etc/pihole/pihole.2.eventHorizon.txt'
 
-counts = {'lc': 0}
-
 # Create the SQLite connection
 conn = sqlite3.connect('/etc/pihole/pihole.db')
 
 with conn:
     c = conn.cursor()
 
-    create_tables()
+    c.execute('DROP TABLE IF EXISTS gravity')
+
+    qt = '''
+    CREATE TABLE IF NOT EXISTS gravity (
+        idx INTEGER PRIMARY KEY ASC,
+        domain text
+    )
+    '''
+    c.execute(qt)
 
     # enable WAL mode
     c.execute('PRAGMA journal_mode=WAL;')
 
     # Parse the log file.
-    for line in open(logfile):
-        line = line.rstrip()
-        counts['lc'] += 1
-
-    if (counts['lc'] % 10000) == 0:
-        conn.commit()
-
-    sql = "INSERT INTO gravity (domain) VALUES (?)"
-    c.execute(sql, (line,))
+    with open(logfile) as f:
+        for line in f:
+            sql = "INSERT INTO gravity (domain) VALUES (?)"
+            c.execute(sql, (line,))

From ae9b5bdffc41bdf2648829316641472a72a5120f Mon Sep 17 00:00:00 2001
From: Dan Schaper <dschaper@ganymeade.com>
Date: Sat, 1 Oct 2016 21:16:31 -0700
Subject: [PATCH 4/4] Comments

---
 advanced/Scripts/gravity_parse.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/advanced/Scripts/gravity_parse.py b/advanced/Scripts/gravity_parse.py
index 29d50be6..42446f4e 100644
--- a/advanced/Scripts/gravity_parse.py
+++ b/advanced/Scripts/gravity_parse.py
@@ -43,16 +43,20 @@
 
 import sqlite3
 
+# Logfile containing unique list of all domains on downloaded lists.
 logfile = '/etc/pihole/pihole.2.eventHorizon.txt'
 
 # Create the SQLite connection
 conn = sqlite3.connect('/etc/pihole/pihole.db')
 
+# Python auto-handle commits, no need to call for commits manually
 with conn:
     c = conn.cursor()
 
+    # Lists have just been downloaded, clear out the existing data
     c.execute('DROP TABLE IF EXISTS gravity')
 
+    # Ready new table for list of domains
     qt = '''
     CREATE TABLE IF NOT EXISTS gravity (
         idx INTEGER PRIMARY KEY ASC,
@@ -64,7 +68,7 @@ with conn:
     # enable WAL mode
     c.execute('PRAGMA journal_mode=WAL;')
 
-    # Parse the log file.
+    # Parse the log file into the database
     with open(logfile) as f:
         for line in f:
             sql = "INSERT INTO gravity (domain) VALUES (?)"