web2py.gluon.cache

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 9 Basic caching classes and methods 10 ================================= 11 12 - Cache - The generic caching object interfacing with the others 13 - CacheInRam - providing caching in ram 14 - CacheInDisk - provides caches on disk 15 16 Memcache is also available via a different module (see gluon.contrib.memcache) 17 18 When web2py is running on Google App Engine, 19 caching will be provided by the GAE memcache 20 (see gluon.contrib.gae_memcache) 21 """ 22 23 import time 24 import portalocker 25 import shelve 26 import thread 27 import os 28 import logging 29 import re 30 31 logger = logging.getLogger("web2py.cache") 32 33 __all__ = ['Cache'] 34 35 36 DEFAULT_TIME_EXPIRE = 300 37 38

39 -class CacheAbstract(object):

40 """ 41 Abstract class for cache implementations. 42 Main function is now to provide referenced api documentation. 43 44 Use CacheInRam or CacheOnDisk instead which are derived from this class. 45 """ 46 47 cache_stats_name = 'web2py_cache_statistics' 48

49 - def __init__(self, request=None):

50 """ 51 Paremeters 52 ---------- 53 request: 54 the global request object 55 """ 56 raise NotImplementedError

57

58 - def __call__(self, key, f, 59 time_expire = DEFAULT_TIME_EXPIRE):

60 """ 61 Tries retrieve the value corresponding to `key` from the cache of the 62 object exists and if it did not expire, else it called the function `f` 63 and stores the output in the cache corresponding to `key`. In the case 64 the output of the function is returned. 65 66 :param key: the key of the object to be store or retrieved 67 :param f: the function, whose output is to be cached 68 :param time_expire: expiration of the cache in microseconds 69 70 - `time_expire` is used to compare the current time with the time when 71 the requested object was last saved in cache. It does not affect 72 future requests. 73 - Setting `time_expire` to 0 or negative value forces the cache to 74 refresh. 75 76 If the function `f` is `None` the cache is cleared. 77 """ 78 raise NotImplementedError

79

80 - def clear(self, regex=None):

81 """ 82 Clears the cache of all keys that match the provided regular expression. 83 If no regular expression is provided, it clears all entries in cache. 84 85 Parameters 86 ---------- 87 regex: 88 if provided, only keys matching the regex will be cleared. 89 Otherwise all keys are cleared. 90 """ 91 92 raise NotImplementedError

93

94 - def increment(self, key, value=1):

95 """ 96 Increments the cached value for the given key by the amount in value 97 98 Parameters 99 ---------- 100 key: 101 key for the cached object to be incremeneted 102 value: 103 amount of the increment (defaults to 1, can be negative) 104 """ 105 raise NotImplementedError

106

107 - def _clear(self, storage, regex):

108 """ 109 Auxiliary function called by `clear` to search and clear cache entries 110 """ 111 r = re.compile(regex) 112 for (key, value) in storage.items(): 113 if r.match(str(key)): 114 del storage[key]

115

116 -class CacheInRam(CacheAbstract):

117 """ 118 Ram based caching 119 120 This is implemented as global (per process, shared by all threads) 121 dictionary. 122 A mutex-lock mechanism avoid conflicts. 123 """ 124 125 locker = thread.allocate_lock() 126 meta_storage = {} 127

128 - def __init__(self, request=None):

129 self.locker.acquire() 130 self.request = request 131 if request: 132 app = request.application 133 else: 134 app = '' 135 if not app in self.meta_storage: 136 self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: { 137 'hit_total': 0, 138 'misses': 0, 139 }} 140 else: 141 self.storage = self.meta_storage[app] 142 self.locker.release()

143

144 - def clear(self, regex=None):

145 self.locker.acquire() 146 storage = self.storage 147 if regex is None: 148 storage.clear() 149 else: 150 self._clear(storage, regex) 151 152 if not CacheAbstract.cache_stats_name in storage.keys(): 153 storage[CacheAbstract.cache_stats_name] = { 154 'hit_total': 0, 155 'misses': 0, 156 } 157 158 self.locker.release()

159

160 - def __call__(self, key, f, 161 time_expire = DEFAULT_TIME_EXPIRE):

162 """ 163 Attention! cache.ram does not copy the cached object. It just stores a reference to it. 164 Turns out the deepcopying the object has some problems: 165 1) would break backward compatibility 166 2) would be limiting because people may want to cache live objects 167 3) would work unless we deepcopy no storage and retrival which would make things slow. 168 Anyway. You can deepcopy explicitly in the function generating the value to be cached. 169 """ 170 171 dt = time_expire 172 173 self.locker.acquire() 174 item = self.storage.get(key, None) 175 if item and f is None: 176 del self.storage[key] 177 self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1 178 self.locker.release() 179 180 if f is None: 181 return None 182 if item and (dt is None or item[0] > time.time() - dt): 183 return item[1] 184 value = f() 185 186 self.locker.acquire() 187 self.storage[key] = (time.time(), value) 188 self.storage[CacheAbstract.cache_stats_name]['misses'] += 1 189 self.locker.release() 190 return value

191

192 - def increment(self, key, value=1):

193 self.locker.acquire() 194 try: 195 if key in self.storage: 196 value = self.storage[key][1] + value 197 self.storage[key] = (time.time(), value) 198 except BaseException, e: 199 self.locker.release() 200 raise e 201 self.locker.release() 202 return value

203 204

205 -class CacheOnDisk(CacheAbstract):

206 """ 207 Disk based cache 208 209 This is implemented as a shelve object and it is shared by multiple web2py 210 processes (and threads) as long as they share the same filesystem. 211 The file is locked wen accessed. 212 213 Disk cache provides persistance when web2py is started/stopped but it slower 214 than `CacheInRam` 215 216 Values stored in disk cache must be pickable. 217 """ 218 219 speedup_checks = set() 220

221 - def __init__(self, request, folder=None):

222 self.request = request 223 224 # Lets test if the cache folder exists, if not 225 # we are going to create it 226 folder = folder or os.path.join(request.folder, 'cache') 227 228 if not os.path.exists(folder): 229 os.mkdir(folder) 230 231 ### we need this because of a possible bug in shelve that may 232 ### or may not lock 233 self.locker_name = os.path.join(folder,'cache.lock') 234 self.shelve_name = os.path.join(folder,'cache.shelve') 235 236 locker, locker_locked = None, False 237 speedup_key = (folder,CacheAbstract.cache_stats_name) 238 if not speedup_key in self.speedup_checks or \ 239 not os.path.exists(self.shelve_name): 240 try: 241 locker = open(self.locker_name, 'a') 242 portalocker.lock(locker, portalocker.LOCK_EX) 243 locker_locked = True 244 storage = shelve.open(self.shelve_name) 245 try: 246 if not storage.has_key(CacheAbstract.cache_stats_name): 247 storage[CacheAbstract.cache_stats_name] = { 248 'hit_total': 0, 249 'misses': 0, 250 } 251 storage.sync() 252 finally: 253 storage.close() 254 self.speedup_checks.add(speedup_key) 255 except ImportError: 256 pass # no module _bsddb, ignoring exception now so it makes a ticket only if used 257 except: 258 logger.error('corrupted file %s, will try delete it!' \ 259 % self.shelve_name) 260 try: 261 os.unlink(self.shelve_name) 262 except IOError: 263 logger.warn('unable to delete file %s' % self.shelve_name) 264 if locker_locked: 265 portalocker.unlock(locker) 266 if locker: 267 locker.close()

268

269 - def clear(self, regex=None):

270 locker = open(self.locker_name,'a') 271 portalocker.lock(locker, portalocker.LOCK_EX) 272 storage = shelve.open(self.shelve_name) 273 try: 274 if regex is None: 275 storage.clear() 276 else: 277 self._clear(storage, regex) 278 if not CacheAbstract.cache_stats_name in storage.keys(): 279 storage[CacheAbstract.cache_stats_name] = { 280 'hit_total': 0, 281 'misses': 0, 282 } 283 storage.sync() 284 finally: 285 storage.close() 286 portalocker.unlock(locker) 287 locker.close()

288

289 - def __call__(self, key, f, 290 time_expire = DEFAULT_TIME_EXPIRE):

291 dt = time_expire 292 293 locker = open(self.locker_name,'a') 294 portalocker.lock(locker, portalocker.LOCK_EX) 295 storage = shelve.open(self.shelve_name) 296 297 item = storage.get(key, None) 298 if item and f is None: 299 del storage[key] 300 301 storage[CacheAbstract.cache_stats_name] = { 302 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1, 303 'misses': storage[CacheAbstract.cache_stats_name]['misses'] 304 } 305 306 storage.sync() 307 308 portalocker.unlock(locker) 309 locker.close() 310 311 if f is None: 312 return None 313 if item and (dt is None or item[0] > time.time() - dt): 314 return item[1] 315 value = f() 316 317 locker = open(self.locker_name,'a') 318 portalocker.lock(locker, portalocker.LOCK_EX) 319 storage[key] = (time.time(), value) 320 321 storage[CacheAbstract.cache_stats_name] = { 322 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'], 323 'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1 324 } 325 326 storage.sync() 327 328 storage.close() 329 portalocker.unlock(locker) 330 locker.close() 331 332 return value

333

334 - def increment(self, key, value=1):

335 locker = open(self.locker_name,'a') 336 portalocker.lock(locker, portalocker.LOCK_EX) 337 storage = shelve.open(self.shelve_name) 338 try: 339 if key in storage: 340 value = storage[key][1] + value 341 storage[key] = (time.time(), value) 342 storage.sync() 343 finally: 344 storage.close() 345 portalocker.unlock(locker) 346 locker.close() 347 return value

348 349

350 -class Cache(object):

351 """ 352 Sets up generic caching, creating an instance of both CacheInRam and 353 CacheOnDisk. 354 In case of GAE will make use of gluon.contrib.gae_memcache. 355 356 - self.ram is an instance of CacheInRam 357 - self.disk is an instance of CacheOnDisk 358 """ 359

360 - def __init__(self, request):

361 """ 362 Parameters 363 ---------- 364 request: 365 the global request object 366 """ 367 # GAE will have a special caching 368 import settings 369 if settings.global_settings.web2py_runtime_gae: 370 from contrib.gae_memcache import MemcacheClient 371 self.ram=self.disk=MemcacheClient(request) 372 else: 373 # Otherwise use ram (and try also disk) 374 self.ram = CacheInRam(request) 375 try: 376 self.disk = CacheOnDisk(request) 377 except IOError: 378 logger.warning('no cache.disk (IOError)') 379 except AttributeError: 380 # normally not expected anymore, as GAE has already 381 # been accounted for 382 logger.warning('no cache.disk (AttributeError)')

383

384 - def __call__(self, 385 key = None, 386 time_expire = DEFAULT_TIME_EXPIRE, 387 cache_model = None):

388 """ 389 Decorator function that can be used to cache any function/method. 390 391 Example:: 392 393 @cache('key', 5000, cache.ram) 394 def f(): 395 return time.ctime() 396 397 When the function f is called, web2py tries to retrieve 398 the value corresponding to `key` from the cache of the 399 object exists and if it did not expire, else it calles the function `f` 400 and stores the output in the cache corresponding to `key`. In the case 401 the output of the function is returned. 402 403 :param key: the key of the object to be store or retrieved 404 :param time_expire: expiration of the cache in microseconds 405 :param cache_model: `cache.ram`, `cache.disk`, or other 406 (like `cache.memcache` if defined). It defaults to `cache.ram`. 407 408 Notes 409 ----- 410 `time_expire` is used to compare the curret time with the time when the 411 requested object was last saved in cache. It does not affect future 412 requests. 413 Setting `time_expire` to 0 or negative value forces the cache to 414 refresh. 415 416 If the function `f` is an action, we suggest using 417 `request.env.path_info` as key. 418 """ 419 if not cache_model: 420 cache_model = self.ram 421 422 def tmp(func): 423 def action(): 424 return cache_model(key, func, time_expire)

425 action.__name___ = func.__name__ 426 action.__doc__ = func.__doc__ 427 return action

428 429 return tmp 430

Source Code for Module web2py.gluon.cache