Database optimizations to speed up web statistics.

Author: suurjaak

README.md

@@ -8,7 +8,7 @@ Three components:
 * listener - logs mouse and keyboard input, can run individually
 * webui - web frontend for statistics and heatmaps, can run individually
 
-Listener and web-UI components can be run separately, or launched from main.
+Listener and web-UI components can be run separately.
 
 Data is kept in an SQLite database, under inputscope/var.
 

inputscope/conf.py

@@ -20,7 +20,7 @@
 
 @author      Erki Suurjaak
 @created     26.03.2015
-@modified    06.05.2015
+@modified    19.05.2015
 ------------------------------------------------------------------------------
 """
 try: import ConfigParser as configparser # Py2
@@ -36,8 +36,8 @@
 
 """Program title, version number and version date."""
 Title = "InputScope"
-Version = "1.0"
-VersionDate = "06.05.2015"
+Version = "1.1a"
+VersionDate = "19.05.2015"
 
 """TCP port of the web user interface."""
 WebHost = "localhost"
@@ -222,16 +222,29 @@
 """Path for application icon file."""
 IconPath = os.path.join(StaticPath, "icon.ico")
 
+"""SQL template for trigger to update day counts."""
+TriggerTemplate = """
+CREATE TRIGGER IF NOT EXISTS on_insert_{0} AFTER INSERT ON {0}
+BEGIN
+  INSERT OR IGNORE INTO counts (type, day, count) VALUES ('{0}', NEW.day, 0);
+  UPDATE counts SET count = count + 1 WHERE type = '{0}' AND day = NEW.day;
+END;"""
+
+"""SQL template for day field index."""
+DayIndexTemplate = "CREATE INDEX IF NOT EXISTS idx_{0}_day ON {0} (day)"
+
 """Statements to execute in database at startup, like CREATE TABLE."""
 DbStatements = (
-  "CREATE TABLE IF NOT EXISTS moves (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP, x INTEGER, y INTEGER)",
-  "CREATE TABLE IF NOT EXISTS clicks (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP, x INTEGER, y INTEGER, button INTEGER)",
-  "CREATE TABLE IF NOT EXISTS scrolls (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP, x INTEGER, y INTEGER, wheel INTEGER)",
-  "CREATE TABLE IF NOT EXISTS keys (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP, key TEXT, realkey TEXT)",
-  "CREATE TABLE IF NOT EXISTS combos (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP, key TEXT, realkey TEXT)",
+  "CREATE TABLE IF NOT EXISTS moves (id INTEGER NOT NULL PRIMARY KEY, day DATE, stamp REAL, x INTEGER, y INTEGER)",
+  "CREATE TABLE IF NOT EXISTS clicks (id INTEGER NOT NULL PRIMARY KEY, day DATE, stamp REAL, x INTEGER, y INTEGER, button INTEGER)",
+  "CREATE TABLE IF NOT EXISTS scrolls (id INTEGER NOT NULL PRIMARY KEY, day DATE, stamp REAL, x INTEGER, y INTEGER, wheel INTEGER)",
+  "CREATE TABLE IF NOT EXISTS keys (id INTEGER NOT NULL PRIMARY KEY, day DATE, stamp REAL, key TEXT, realkey TEXT)",
+  "CREATE TABLE IF NOT EXISTS combos (id INTEGER NOT NULL PRIMARY KEY, day DATE, stamp REAL, key TEXT, realkey TEXT)",
   "CREATE TABLE IF NOT EXISTS app_events (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP DEFAULT (DATETIME('now', 'localtime')), type TEXT)",
   "CREATE TABLE IF NOT EXISTS screen_sizes (id INTEGER NOT NULL PRIMARY KEY, dt TIMESTAMP DEFAULT (DATETIME('now', 'localtime')), x INTEGER, y INTEGER)",
-)
+  "CREATE TABLE IF NOT EXISTS counts (id INTEGER NOT NULL PRIMARY KEY, type TEXT, day DATETIME, count INTEGER, UNIQUE(type, day))",
+) + tuple(TriggerTemplate.format(x) for x in ["moves", "clicks", "scrolls", "keys", "combos"]
+) + tuple(DayIndexTemplate.format(x) for x in ["moves", "clicks", "scrolls", "keys", "combos"])
 
 
 def init(filename=ConfigPath):

inputscope/listener.py

@@ -6,13 +6,14 @@
 
 @author      Erki Suurjaak
 @created     06.04.2015
-@modified    07.05.2015
+@modified    19.05.2015
 """
 from __future__ import print_function
 import datetime
 import Queue
 import sys
 import threading
+import time
 import pykeyboard
 import pymouse
 
@@ -102,7 +103,7 @@ def stop(self):
         db.close()
 
     def handle(self, **kwargs):
-        kwargs["dt"] = datetime.datetime.now()
+        kwargs.update(day=datetime.date.today(), stamp=time.time())
         self.inqueue.put(kwargs)
 
 

inputscope/main.py

@@ -5,7 +5,7 @@
 
 @author      Erki Suurjaak
 @created     05.05.2015
-@modified    04.05.2015
+@modified    19.05.2015
 """
 import multiprocessing
 import multiprocessing.forking
@@ -122,7 +122,7 @@ def OnInit(self):
         self.trayicon.Bind(wx.EVT_TASKBAR_RIGHT_DOWN, self.OnOpenMenu)
         self.frame_console.Bind(wx.EVT_CLOSE, self.OnToggleConsole)
 
-        self.model.log_resolution(wx.GetDisplaySize())
+        wx.CallAfter(self.model.log_resolution, wx.GetDisplaySize())
         wx.CallAfter(self.model.start)
         return True # App.OnInit returns whether processing should continue
 

inputscope/var/inputscope.db

@@ -6,7 +6,7 @@
 
 @author      Erki Suurjaak
 @created     06.04.2015
-@modified    06.05.2015
+@modified    18.05.2015
 """
 import collections
 import datetime
@@ -39,8 +39,9 @@ def server_static(filepath):
 @route("/mouse/<table>/<day>")
 def mouse(table, day=None):
     """Handler for showing mouse statistics for specified type and day."""
-    where = (("DATE(dt)", day),) if day else ()
-    events = db.fetch(table, where=where, order="dt")
+    where = (("day", day),) if day else ()
+    events = db.fetch(table, where=where, order="day")
+    for e in events: e["dt"] = datetime.datetime.fromtimestamp(e["stamp"])
     stats, positions, events = stats_mouse(events, table)
     return bottle.template("mouse.tpl", locals(), conf=conf)
 
@@ -50,12 +51,13 @@ def mouse(table, day=None):
 def keyboard(table, day=None):
     """Handler for showing the keyboard statistics page."""
     cols, group = "realkey AS key, COUNT(*) AS count", "realkey"
-    where = (("DATE(dt)", day),) if day else ()
+    where = (("day", day),) if day else ()
     counts_display = counts = db.fetch(table, cols, where, group, "count DESC")
     if "combos" == table:
         counts_display = db.fetch(table, "key, COUNT(*) AS count", where,
                                   "key", "count DESC")
-    events = db.fetch(table, where=where, order="id")
+    events = db.fetch(table, where=where, order="stamp")
+    for e in events: e["dt"] = datetime.datetime.fromtimestamp(e["stamp"])
     stats, collatedevents = stats_keyboard(events, table)
     return bottle.template("keyboard.tpl", locals(), conf=conf)
 
@@ -64,23 +66,22 @@ def keyboard(table, day=None):
 def inputindex(input):
     """Handler for showing keyboard or mouse page with day and total links."""
     stats = {}
-    countminmax = "COUNT(*) AS count, MIN(DATE(dt)) AS first, MAX(DATE(dt)) AS last"
-    dtcols = "DATE(dt) AS day, COUNT(*) AS count"
+    countminmax = "SUM(count) AS count, MIN(day) AS first, MAX(day) AS last"
     tables = ("moves", "clicks", "scrolls") if "mouse" == input else ("keys", "combos")
     for table in tables:
-        stats[table] = db.fetchone(table, countminmax)
-        stats[table]["days"] = db.fetch(table, dtcols, group="day", order="day")
+        stats[table] = db.fetchone("counts", countminmax, type=table)
+        stats[table]["days"] = db.fetch("counts", order="day DESC", type=table)
     return bottle.template("input.tpl", locals(), conf=conf)
 
 
 @route("/")
 def index():
     """Handler for showing the GUI index page."""
     stats = {"mouse": {"count": 0}, "keyboard": {"count": 0}}
-    countminmax = "COUNT(*) AS count, MIN(DATE(dt)) AS first, MAX(DATE(dt)) AS last"
+    countminmax = "SUM(count) AS count, MIN(day) AS first, MAX(day) AS last"
     tables = {"keyboard": ("keys", "combos"), "mouse": ("moves", "clicks", "scrolls")}
     for input, table in [(x, t) for x, tt in tables.items() for t in tt]:
-        row = db.fetchone(table, countminmax)
+        row = db.fetchone("counts", countminmax, type=table)
         if not row["count"]: continue # for input, table
         stats[input]["count"] += row["count"]
         for func, key in [(min, "first"), (max, "last")]:
@@ -120,7 +121,7 @@ def stats_keyboard(events, table):
          sum(deltas, datetime.timedelta()) / len(deltas)),
     ] if "combos" == table else [
         ("Keys per hour",
-         int(len(events) / (timedelta_seconds(events[-1]["dt"] - events[0]["dt"]) / 3600))),
+         int(3600 * len(events) / timedelta_seconds(events[-1]["dt"] - events[0]["dt"]))),
         ("Average interval between keys",
          sum(deltas, datetime.timedelta()) / len(deltas)),
         ("Typing sessions (key interval < %ss)" % UNBROKEN_DELTA.seconds,
@@ -176,40 +177,33 @@ def stats_mouse(events, table):
                   (distance * conf.PixelLength, conf.PixelLength * 1000)),
                  ("Average speed", "%.1f pixels per second" % (distance / (seconds or 1))),
                  ("", "%.4f meters per second" %
-                  (distance * conf.PixelLength / (seconds or 1))),
-                ]
+                  (distance * conf.PixelLength / (seconds or 1))), ]
     elif "scrolls" == table:
         counts = collections.Counter(e["wheel"] for e in events)
         stats = [("Scrolls per hour", 
                   int(len(events) / (timedelta_seconds(events[-1]["dt"] - events[0]["dt"]) / 3600 or 1))),
                  ("Average interval", sum(deltas, datetime.timedelta()) / (len(deltas) or 1)),
                  ("Scrolls down", counts[-1]),
-                 ("Scrolls up", counts[1]),
-                ]
+                 ("Scrolls up", counts[1]), ]
     elif "clicks" == table:
-        "average distance between clicks"
         counts = collections.Counter(e["button"] for e in events)
         NAMES = {1: "Left", 2: "Right", 3: "Middle"}
         stats = [("Clicks per hour", 
                   int(len(events) / (timedelta_seconds(events[-1]["dt"] - events[0]["dt"]) / 3600 or 1))),
                  ("Average interval between clicks",
                   sum(deltas, datetime.timedelta()) / (len(deltas) or 1)),
                  ("Average distance between clicks",
-                  "%.1f pixels" % (distance / (len(events) or 1))),
-                ]
+                  "%.1f pixels" % (distance / (len(events) or 1))), ]
         for k, v in sorted(counts.items()):
             stats += [("%s button clicks" % NAMES.get(k, "%s." % k), v)]
     return stats, positions, events
 
 
 def timedelta_seconds(timedelta):
     """Returns the total timedelta duration in seconds."""
-    if hasattr(timedelta, "total_seconds"):
-        result = timedelta.total_seconds()
-    else: # Python 2.6 compatibility
-        result = timedelta.days * 24 * 3600 + timedelta.seconds + \
-                 timedelta.microseconds / 1000000.
-    return result
+    return (timedelta.total_seconds() if hasattr(timedelta, "total_seconds")
+            else timedelta.days * 24 * 3600 + timedelta.seconds +
+                 timedelta.microseconds / 1000000.)
 
 
 def init():