Change eviction policy to use new-style string formatting and add test for custom eviction policy

Author: grantjenks

diskcache/core.py

@@ -95,23 +95,23 @@ def __repr__(self):
             ' Cache (store_time)'
         ),
         'get': None,
-        'cull': 'SELECT %s FROM Cache ORDER BY store_time LIMIT ?',
+        'cull': 'SELECT {fields} FROM Cache ORDER BY store_time LIMIT ?',
     },
     'least-recently-used': {
         'init': (
             'CREATE INDEX IF NOT EXISTS Cache_access_time ON'
             ' Cache (access_time)'
         ),
-        'get': 'access_time = ((julianday("now") - 2440587.5) * 86400.0)',
-        'cull': 'SELECT %s FROM Cache ORDER BY access_time LIMIT ?',
+        'get': 'access_time = {now}',
+        'cull': 'SELECT {fields} FROM Cache ORDER BY access_time LIMIT ?',
     },
     'least-frequently-used': {
         'init': (
             'CREATE INDEX IF NOT EXISTS Cache_access_count ON'
             ' Cache (access_count)'
         ),
         'get': 'access_count = access_count + 1',
-        'cull': 'SELECT %s FROM Cache ORDER BY access_count LIMIT ?',
+        'cull': 'SELECT {fields} FROM Cache ORDER BY access_count LIMIT ?',
     },
 }
 
@@ -739,21 +739,20 @@ def _cull(self, now, sql, cleanup, limit=None):
 
         # Evict keys by policy.
 
-        select_policy_template = EVICTION_POLICY[self.eviction_policy]['cull']
+        select_policy = EVICTION_POLICY[self.eviction_policy]['cull']
 
-        if select_policy_template is None or self.volume() < self.size_limit:
+        if select_policy is None or self.volume() < self.size_limit:
             return
 
-        select_policy = select_policy_template % 'filename'
-
-        rows = sql(select_policy, (cull_limit,)).fetchall()
+        select_filename = select_policy.format(fields='filename', now=now)
+        rows = sql(select_filename, (cull_limit,)).fetchall()
 
         if rows:
-            delete_policy = (
+            delete = (
                 'DELETE FROM Cache WHERE rowid IN (%s)'
-                % (select_policy_template % 'rowid')
+                % (select_policy.format(fields='rowid', now=now))
             )
-            sql(delete_policy, (cull_limit,))
+            sql(delete, (cull_limit,))
 
             for filename, in rows:
                 cleanup(filename)
@@ -868,7 +867,10 @@ def incr(self, key, delta=1, default=0):
 
             columns = 'store_time = ?, value = ?'
             update_column = EVICTION_POLICY[self.eviction_policy]['get']
-            columns += '' if update_column is None else ', ' + update_column
+
+            if update_column is not None:
+                columns += ', ' + update_column.format(now=now)
+
             update = 'UPDATE Cache SET %s WHERE rowid = ?' % columns
             sql(update, (now, value, rowid))
 
@@ -918,7 +920,6 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False):
         """
         db_key, raw = self._disk.put(key)
         update_column = EVICTION_POLICY[self.eviction_policy]['get']
-        update = 'UPDATE Cache SET %s WHERE rowid = ?'
         select = (
             'SELECT rowid, expire_time, tag, mode, filename, value'
             ' FROM Cache WHERE key = ? AND raw = ?'
@@ -950,7 +951,6 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False):
                     raise
 
         else:  # Slow path, transaction required.
-
             cache_hit = (
                 'UPDATE Settings SET value = value + 1 WHERE key = "hits"'
             )
@@ -983,8 +983,11 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False):
                 if self.statistics:
                     sql(cache_hit)
 
+                now = time.time()
+                update = 'UPDATE Cache SET %s WHERE rowid = ?'
+
                 if update_column is not None:
-                    sql(update % update_column, (rowid,))
+                    sql(update % update_column.format(now=now), (rowid,))
 
         if expire_time and tag:
             return (value, db_expire_time, db_tag)
@@ -1568,25 +1571,25 @@ def cull(self):
 
         # Remove items by policy.
 
-        select_policy_template = EVICTION_POLICY[self.eviction_policy]['cull']
+        select_policy = EVICTION_POLICY[self.eviction_policy]['cull']
 
-        if select_policy_template is None:
+        if select_policy is None:
             return
 
-        select_policy = select_policy_template % 'filename'
+        select_filename = select_policy.format(fields='filename', now=now)
 
         try:
             while self.volume() > self.size_limit:
                 with self._transact() as (sql, cleanup):
-                    rows = sql(select_policy, (10,)).fetchall()
+                    rows = sql(select_filename, (10,)).fetchall()
 
                     if not rows:
                         break
 
                     count += len(rows)
                     delete = (
                         'DELETE FROM Cache WHERE rowid IN (%s)'
-                        % (select_policy_template % 'rowid')
+                        % select_policy.format(fields='rowid', now=now)
                     )
                     sql(delete, (10,))
 

tests/test_core.py

@@ -1448,6 +1448,43 @@ def test_rsync():
     shutil.rmtree('tmp', ignore_errors=True)
 
 
+@setup_cache
+def test_custom_eviction_policy(cache):
+    dc.EVICTION_POLICY['lru-gt-1s'] = {
+        'init': (
+            'CREATE INDEX IF NOT EXISTS Cache_access_time ON'
+            ' Cache (access_time)'
+        ),
+        'get': 'access_time = {now}',
+        'cull': (
+            'SELECT {fields} FROM Cache'
+            ' WHERE access_time < ({now} - 1)'
+            ' ORDER BY access_time LIMIT ?'
+        ),
+    }
+
+    size_limit = int(1e5)
+
+    cache.reset('eviction_policy', 'lru-gt-1s')
+    cache.reset('size_limit', size_limit)
+
+    for count in range(100, 150):
+        cache[count] = str(count) * 500
+
+    size = cache.volume()
+    assert size > size_limit
+    assert cache.cull() == 0
+    assert size == cache.volume()
+
+    for count in range(100, 150):
+        assert cache[count] == str(count) * 500
+
+    time.sleep(1.1)
+
+    assert cache.cull() == 20
+    assert cache.volume() < size_limit
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()