Package omeroweb :: Package webgateway :: Module webgateway_cache
[hide private]
[frames] | no frames]

Source Code for Module omeroweb.webgateway.webgateway_cache

  1  # 
  2  # webgateway/webgateway_cache - web cache handler for webgateway 
  3  #  
  4  # Copyright (c) 2008, 2009 Glencoe Software, Inc. All rights reserved. 
  5  #  
  6  # This software is distributed under the terms described by the LICENCE file 
  7  # you can find at the root of the distribution bundle, which states you are 
  8  # free to use it only for non commercial purposes. 
  9  # If the file is missing please request a copy by contacting 
 10  # jason@glencoesoftware.com. 
 11  # 
 12  # Author: Carlos Neves <carlos(at)glencoesoftware.com> 
 13   
 14  from django.conf import settings 
 15  import omero 
 16  import logging 
 17  from random import random 
 18  import datetime 
 19   
 20   
 21  logger = logging.getLogger('cache') 
 22   
 23  import struct, time, os, re, shutil, stat 
 24  size_of_double = len(struct.pack('d',0)) 
 25  string_type = type('') 
 26   
 27  CACHE=getattr(settings, 'WEBGATEWAY_CACHE', None) 
 28  TMPROOT=getattr(settings, 'WEBGATEWAY_TMPROOT', None) 
 29  THUMB_CACHE_TIME = 3600 # 1 hour 
 30  THUMB_CACHE_SIZE = 20*1024 # KB == 20MB 
 31  IMG_CACHE_TIME= 3600 # 1 hour 
 32  IMG_CACHE_SIZE = 512*1024 # KB == 512MB 
 33  JSON_CACHE_TIME= 3600 # 1 hour 
 34  JSON_CACHE_SIZE = 1*1024 # KB == 1MB 
 35  TMPDIR_TIME = 3600 * 12 # 12 hours 
 36   
37 -class CacheBase (object): #pragma: nocover
38 """ 39 Caching base class - extended by L{FileCache} for file-based caching. 40 Methods of this base class return None or False providing a no-caching implementation if needed 41 """ 42
43 - def __init__ (self):
44 """ not implemented """ 45 pass
46
47 - def get (self, k):
48 return None
49
50 - def set (self, k, v, t=0, invalidateGroup=None):
51 return False
52
53 - def delete (self, k):
54 return False
55
56 - def wipe (self):
57 return False
58
59 -class FileCache(CacheBase):
60 """ 61 Implements file-based caching within the directory specified in constructor. 62 """ 63 _purge_holdoff = 4 64
65 - def __init__(self, dir, timeout=60, max_entries=0, max_size=0):
66 """ 67 Initialises the class. 68 69 @param dir: Path to directory to place cached files. 70 @param timeout: Cache timeout in secs 71 @param max_entries: If specified, limits number of items to cache 72 @param max_size: Maxium size of cache in KB 73 """ 74 75 super(FileCache, self).__init__() 76 self._dir = dir 77 self._max_entries = max_entries 78 self._max_size = max_size 79 self._last_purge = 0 80 self._default_timeout=timeout 81 if not os.path.exists(self._dir): 82 self._createdir()
83 #
84 - def add(self, key, value, timeout=None, invalidateGroup=None):
85 """ 86 Adds data to cache, returning False if already cached. Otherwise delegating to L{set} 87 88 @param key: Unique key for cache 89 @param value: Value to cache - must be String 90 @param timeout: Optional timeout - otherwise use default 91 @param invalidateGroup: Not used? 92 """ 93 94 if self.has_key(key): 95 return False 96 97 self.set(key, value, timeout, invalidateGroup=invalidateGroup) 98 return True
99
100 - def get(self, key, default=None):
101 """ 102 Gets data from cache 103 104 @param key: cache key 105 @param default: default value to return 106 @return: cache data or default if timout has passed 107 """ 108 fname = self._key_to_file(key) 109 try: 110 f = open(fname, 'rb') 111 exp = struct.unpack('d',f.read(size_of_double))[0] 112 now = time.time() 113 if exp < now: 114 f.close() 115 self._delete(fname) 116 else: 117 return f.read() 118 except (IOError, OSError, EOFError, struct.error): 119 pass 120 return default
121
122 - def set(self, key, value, timeout=None, invalidateGroup=None):
123 """ 124 Adds data to cache, overwriting if already cached. 125 126 @param key: Unique key for cache 127 @param value: Value to cache - must be String 128 @param timeout: Optional timeout - otherwise use default 129 @param invalidateGroup: Not used? 130 """ 131 132 if type(value) != string_type: 133 raise ValueError("%s not a string, can't cache" % type(value)) 134 fname = self._key_to_file(key) 135 dirname = os.path.dirname(fname) 136 137 if timeout is None: 138 timeout = self._default_timeout 139 140 if self._full(): 141 # Maybe we already have this one cached, and we need the space 142 try: 143 self._delete(fname) 144 except OSError: 145 pass 146 if self._full(): 147 return 148 149 try: 150 if not os.path.exists(dirname): 151 os.makedirs(dirname) 152 153 f = open(fname, 'wb') 154 exp = time.time() + timeout + (timeout / 5 * random()) 155 f.write(struct.pack('d', exp)) 156 f.write(value) 157 except (IOError, OSError): #pragma: nocover 158 pass
159
160 - def delete(self, key):
161 """ 162 Attempt to delete the cache data referenced by key 163 @param key: Cache key 164 """ 165 166 try: 167 self._delete(self._key_to_file(key)) 168 except (IOError, OSError): #pragma: nocover 169 pass
170
171 - def _delete(self, fname):
172 """ 173 Tries to delete the data at the specified absolute file path 174 175 @param fname: File name of data to delete 176 """ 177 178 logger.debug('requested delete for "%s"' % fname) 179 if os.path.isdir(fname): 180 shutil.rmtree(fname, ignore_errors=True) 181 else: 182 os.remove(fname) 183 try: 184 # Remove the parent subdirs if they're empty 185 dirname = os.path.dirname(fname) 186 while dirname != self._dir: 187 os.rmdir(dirname) 188 dirname = os.path.dirname(fname) 189 except (IOError, OSError): 190 pass
191
192 - def wipe (self):
193 """ Deletes everything in the cache """ 194 195 shutil.rmtree(self._dir) 196 self._createdir() 197 return True
198
199 - def _check_entry (self, fname):
200 """ 201 Verifies if a specific cache entry (provided as absolute file path) is expired. 202 If expired, it gets deleted and method returns false. 203 If not expired, returns True. 204 205 @param fname: File path 206 """ 207 try: 208 f = open(fname, 'rb') 209 exp = struct.unpack('d',f.read(size_of_double))[0] 210 now = time.time() 211 if exp < now: 212 f.close() 213 self._delete(fname) 214 return False 215 else: 216 return True 217 except (IOError, OSError, EOFError, struct.error): #pragma: nocover 218 return False
219
220 - def has_key(self, key):
221 """ 222 Returns true if the cache has the specified key 223 @param key: Key to look for. 224 @rtype: Boolean 225 """ 226 fname = self._key_to_file(key) 227 return self._check_entry(fname)
228
229 - def _du (self):
230 """ 231 Disk Usage count on the filesystem the cache is based at 232 233 @rtype: int 234 @return: the current usage, in KB 235 """ 236 return int(os.popen('du -sk %s' % os.path.join(os.getcwd(),self._dir)).read().split('\t')[0].strip())
237
238 - def _full(self, _on_retry=False):
239 """ 240 Checks whether the cache is full, either because we have exceeded max number of entries or 241 the cache space is full. 242 243 @param _on_retry: Flag allows calling this method again after purge() without recursion 244 @return: True if cache is full 245 @rtype: Boolean 246 """ 247 248 # Check nr of entries 249 if self._max_entries: 250 try: 251 x = int(os.popen('find %s -type f | wc -l' % self._dir).read().strip()) 252 if x >= self._max_entries: 253 if not _on_retry: 254 self._purge() 255 return self._full(True) 256 logger.warn('caching limits reached on %s: max entries %d' % (self._dir, self._max_entries)) 257 return True 258 except ValueError: #pragma: nocover 259 logger.error('Counting cache entries failed') 260 # Check for space usage 261 if self._max_size: 262 try: 263 x = self._du() 264 if x >= self._max_size: 265 if not _on_retry: 266 self._purge() 267 return self._full(True) 268 logger.warn('caching limits reached on %s: max size %d' % (self._dir, self._max_size)) 269 return True 270 except ValueError: #pragma: nocover 271 logger.error('Counting cache size failed') 272 return False
273
274 - def _purge (self):
275 """ 276 Iterate the whole cache structure searching and cleaning expired entries. 277 this method may be expensive, so only call it when really necessary. 278 """ 279 now = time.time() 280 if now-self._last_purge < self._purge_holdoff: 281 return 282 self._last_purge = now 283 284 logger.debug('entering purge') 285 count = 0 286 for p,_,files in os.walk(self._dir): 287 for f in files: 288 if not self._check_entry(os.path.join(p, f)): 289 count += 1 290 logger.debug('purge finished, removed %d files' % count)
291
292 - def _createdir(self):
293 """ 294 Creates a directory for the root dir of the cache. 295 """ 296 try: 297 os.makedirs(self._dir) 298 except OSError: #pragma: nocover 299 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
300
301 - def _key_to_file(self, key):
302 """ 303 Uses the key to construct an absolute path to the cache data. 304 @param key: Cache key 305 @return: Path 306 @rtype: String 307 """ 308 309 if key.find('..') > 0 or key.startswith('/'): 310 raise ValueError('Invalid value for cache key: "%s"' % key) 311 return os.path.join(self._dir, key)
312
313 - def _get_num_entries(self):
314 """ 315 Returns the number of files in the cache 316 @rtype: int 317 """ 318 count = 0 319 for _,_,files in os.walk(self._dir): 320 count += len(files) 321 return count
322 _num_entries = property(_get_num_entries)
323 324 FN_REGEX = re.compile('[#$,|]')
325 -class WebGatewayCache (object):
326 """ 327 Caching class for webgateway. 328 """ 329
330 - def __init__ (self, backend=None, basedir=CACHE):
331 """ 332 Initialises cache 333 334 @param backend: The cache class to use for caching. E.g. L{FileCache} 335 @param basedir: The base location for all caches. Sub-dirs created for json/ img/ thumb/ 336 """ 337 338 self._basedir = basedir 339 self._lastlock = None 340 if backend is None or basedir is None: 341 self._json_cache = CacheBase() 342 self._img_cache = CacheBase() 343 self._thumb_cache = CacheBase() 344 else: 345 self._json_cache = backend(dir=os.path.join(basedir,'json'), 346 timeout=JSON_CACHE_TIME, max_entries=0, max_size=JSON_CACHE_SIZE) 347 self._img_cache = backend(dir=os.path.join(basedir,'img'), 348 timeout=IMG_CACHE_TIME, max_entries=0, max_size=IMG_CACHE_SIZE) 349 self._thumb_cache = backend(dir=os.path.join(basedir,'thumb'), 350 timeout=THUMB_CACHE_TIME, max_entries=0, max_size=THUMB_CACHE_SIZE)
351
352 - def _updateCacheSettings (self, cache, timeout=None, max_entries=None, max_size=None):
353 """ 354 Updates the timeout, max_entries and max_size (if specified) for the given cache 355 356 @param cache: Cache or caches to update. 357 @type cache: L{CacheBase} or list of caches 358 """ 359 360 if isinstance(cache, CacheBase): 361 cache = (cache,) 362 for c in cache: 363 if timeout is not None: 364 c._default_timeout = timeout 365 if max_entries is not None: 366 c._max_entries = max_entries 367 if max_size is not None: 368 c._max_size = max_size
369
370 - def __del__ (self):
371 """ 372 Tries to remove the lock on this cache. 373 """ 374 if self._lastlock: 375 try: 376 logger.debug('removing cache lock file on __del__') 377 os.remove(self._lastlock) 378 except: 379 pass 380 self._lastlock = None
381
382 - def tryLock (self):
383 """ 384 simple lock mechanisn to avoid multiple processes on the same cache to 385 step on each other's toes. 386 387 @rtype: boolean 388 @return: True if we created a lockfile or already had it. False otherwise. 389 """ 390 lockfile = os.path.join(self._basedir, '%s_lock' % datetime.datetime.now().strftime('%Y%m%d_%H%M')) 391 if self._lastlock: 392 if lockfile == self._lastlock: 393 return True 394 try: 395 os.remove(self._lastlock) 396 except: 397 pass 398 self._lastlock = None 399 try: 400 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL) 401 os.close(fd) 402 self._lastlock = lockfile 403 return True 404 except OSError: 405 return False
406
407 - def handleEvent (self, client_base, e):
408 """ 409 Handle one event from blitz.onEventLogs. 410 411 Meant to be overridden, this implementation just logs. 412 413 @param client_base: TODO: docs! 414 @param e: 415 """ 416 logger.debug('## %s#%i %s user #%i group #%i(%i)' % (e.entityType.val, 417 e.entityId.val, 418 e.action.val, 419 e.details.owner.id.val, 420 e.details.group.id.val, 421 e.event.id.val))
422
423 - def eventListener (self, client_base, events):
424 """ 425 handle events coming our way from blitz.onEventLogs. 426 427 Because all processes will be listening to the same events, we use a simple file 428 lock mechanism to make sure the first process to get the event will be the one 429 handling things from then on. 430 431 @param client_base: TODO: docs! 432 @param events: 433 """ 434 for e in events: 435 if self.tryLock(): 436 self.handleEvent(client_base, e) 437 else: 438 logger.debug("## ! ignoring event %s" % str(e.event.id.val))
439
440 - def clear (self):
441 """ 442 Clears all the caches. 443 """ 444 self._json_cache.wipe() 445 self._img_cache.wipe() 446 self._thumb_cache.wipe()
447
448 - def _cache_set (self, cache, key, obj):
449 """ Calls cache.set(key, obj) """ 450 451 logger.debug(' set: %s' % key) 452 cache.set(key, obj)
453
454 - def _cache_clear (self, cache, key):
455 """ Calls cache.delete(key) """ 456 457 logger.debug(' clear: %s' % key) 458 cache.delete(key)
459
460 - def invalidateObject (self, client_base, user_id, obj):
461 """ 462 Invalidates all caches for this particular object 463 464 @param client_base: The server_id 465 @param user_id: OMERO user ID to partition caching upon 466 @param obj: The object wrapper. E.g. L{omero.gateway.ImageWrapper} 467 """ 468 469 if obj.OMERO_CLASS == 'Image': 470 self.clearImage(None, client_base, user_id, obj) 471 else: 472 logger.debug('unhandled object type: %s' % obj.OMERO_CLASS) 473 self.clearJson(client_base, obj)
474 475 ## 476 # Thumb 477
478 - def _thumbKey (self, r, client_base, user_id, iid, size):
479 """ 480 Generates a string key for caching the thumbnail, based on the above parameters 481 482 @param r: not used 483 @param client_base: server-id, forms stem of the key 484 @param user_id: OMERO user ID to partition caching upon 485 @param iid: image ID 486 @param size: size of the thumbnail - tuple. E.g. (100,) 487 """ 488 489 if size is not None and len(size): 490 return 'thumb_user_%s/%s/%s/%s' % (client_base, str(iid), user_id, 'x'.join([str(x) for x in size])) 491 else: 492 return 'thumb_user_%s/%s/%s' % (client_base, str(iid), user_id)
493
494 - def setThumb (self, r, client_base, user_id, iid, obj, size=()):
495 """ 496 Puts thumbnail into cache. 497 498 @param r: for cache key - Not used? 499 @param client_base: server_id for cache key 500 @param user_id: OMERO user ID to partition caching upon 501 @param iid: image ID for cache key 502 @param obj: Data to cache 503 @param size: Size used for cache key. Tuple 504 """ 505 506 k = self._thumbKey(r, client_base, user_id, iid, size) 507 self._cache_set(self._thumb_cache, k, obj) 508 return True
509
510 - def getThumb (self, r, client_base, user_id, iid, size=()):
511 """ 512 Gets thumbnail from cache. 513 514 @param r: for cache key - Not used? 515 @param client_base: server_id for cache key 516 @param user_id: OMERO user ID to partition caching upon 517 @param iid: image ID for cache key 518 @param size: Size used for cache key. Tuple 519 @return: Cached data or None 520 @rtype: String 521 """ 522 523 k = self._thumbKey(r, client_base, user_id, iid, size) 524 r = self._thumb_cache.get(k) 525 if r is None: 526 logger.debug(' fail: %s' % k) 527 else: 528 logger.debug('cached: %s' % k) 529 return r
530
531 - def clearThumb (self, r, client_base, user_id, iid, size=None):
532 """ 533 Clears thumbnail from cache. 534 535 @param r: for cache key - Not used? 536 @param client_base: server_id for cache key 537 @param user_id: OMERO user ID to partition caching upon 538 @param iid: image ID for cache key 539 @param size: Size used for cache key. Tuple 540 @return: True 541 """ 542 k = self._thumbKey(r, client_base, user_id, iid, size) 543 self._cache_clear(self._thumb_cache, k) 544 return True
545 546 ## 547 # Image 548
549 - def _imageKey (self, r, client_base, img, z=0, t=0):
550 """ 551 Returns a key for caching the Image, based on parameters above, including rendering settings 552 specified in the http request. 553 554 @param r: http request - get rendering params 'c', 'm', 'p' 555 @param client_base: server_id for cache key 556 @param img: L{omero.gateway.ImageWrapper} for ID 557 @param obj: Data to cache 558 @param size: Size used for cache key. Tuple 559 """ 560 561 iid = img.getId() 562 if r: 563 r = r.REQUEST 564 c = FN_REGEX.sub('-',r.get('c', '')) 565 m = r.get('m', '') 566 p = r.get('p', '') 567 if p and not isinstance(omero.gateway.ImageWrapper.PROJECTIONS.get(p, -1), 568 omero.constants.projection.ProjectionType): #pragma: nocover 569 p = '' 570 q = r.get('q', '') 571 region = r.get('region', '') 572 tile = r.get('tile', '') 573 rv = 'img_%s/%s/%%s-c%s-m%s-q%s-r%s-t%s' % (client_base, str(iid), c, m, q, region, tile) 574 if p: 575 return rv % ('%s-%s' % (p, str(t))) 576 else: 577 return rv % ('%sx%s' % (str(z), str(t))) 578 else: 579 return 'img_%s/%s/' % (client_base, str(iid))
580
581 - def setImage (self, r, client_base, img, z, t, obj, ctx=''):
582 """ 583 Puts image data into cache. 584 585 @param r: http request for cache key 586 @param client_base: server_id for cache key 587 @param img: ImageWrapper for cache key 588 @param z: Z index for cache key 589 @param t: T index for cache key 590 @param obj: Data to cache 591 @param ctx: Additional string for cache key 592 """ 593 594 k = self._imageKey(r, client_base, img, z, t) + ctx 595 self._cache_set(self._img_cache, k, obj) 596 return True
597
598 - def getImage (self, r, client_base, img, z, t, ctx=''):
599 """ 600 Gets image data from cache. 601 602 @param r: http request for cache key 603 @param client_base: server_id for cache key 604 @param img: ImageWrapper for cache key 605 @param z: Z index for cache key 606 @param t: T index for cache key 607 @param ctx: Additional string for cache key 608 @return: Image data 609 @rtype: String 610 """ 611 k = self._imageKey(r, client_base, img, z, t) + ctx 612 r = self._img_cache.get(k) 613 if r is None: 614 logger.debug(' fail: %s' % k) 615 else: 616 logger.debug('cached: %s' % k) 617 return r
618
619 - def clearImage (self, r, client_base, user_id, img):
620 """ 621 Clears image data from cache using default rendering settings (r=None) T and Z indexes ( = 0). 622 TODO: Doesn't clear any data stored WITH r, t, or z specified in cache key? 623 Also clears thumbnail (but not thumbs with size specified) and json data for this image. 624 625 @param r: http request for cache key 626 @param client_base: server_id for cache key 627 @param user_id: OMERO user ID to partition caching upon 628 @param img: ImageWrapper for cache key 629 @param obj: Data to cache 630 @param rtype: True 631 """ 632 633 k = self._imageKey(None, client_base, img) 634 self._cache_clear(self._img_cache, k) 635 # do the thumb too 636 self.clearThumb(r, client_base, user_id, img.getId()) 637 # and json data 638 self.clearJson(client_base, img) 639 return True
640
641 - def setSplitChannelImage (self, r, client_base, img, z, t, obj):
642 """ Calls L{setImage} with '-sc' context """ 643 return self.setImage(r, client_base, img, z, t, obj, '-sc')
644
645 - def getSplitChannelImage (self, r, client_base, img, z, t):
646 """ 647 Calls L{getImage} with '-sc' context 648 @rtype: String 649 """ 650 return self.getImage(r, client_base, img, z, t, '-sc')
651
652 - def setOmeTiffImage (self, r, client_base, img, obj):
653 """ Calls L{setImage} with '-ometiff' context """ 654 return self.setImage(r, client_base, img, 0, 0, obj, '-ometiff')
655
656 - def getOmeTiffImage (self, r, client_base, img):
657 """ 658 Calls L{getImage} with '-ometiff' context 659 @rtype: String 660 """ 661 return self.getImage(r, client_base, img, 0, 0, '-ometiff')
662 663 ## 664 # hierarchies (json) 665
666 - def _jsonKey (self, r, client_base, obj, ctx=''):
667 """ 668 Creates a cache key for storing json data based on params above. 669 670 @param r: http request - not used 671 @param client_base: server_id 672 @param obj: ObjectWrapper 673 @param ctx: Additional string for cache key 674 @return: Cache key 675 @rtype: String 676 """ 677 678 if obj: 679 return 'json_%s/%s_%s/%s' % (client_base, obj.OMERO_CLASS, obj.id, ctx) 680 else: 681 return 'json_%s/single/%s' % (client_base, ctx)
682
683 - def clearJson (self, client_base, obj):
684 """ 685 Only handles Dataset obj, calling L{clearDatasetContents} 686 """ 687 logger.debug('clearjson') 688 if obj.OMERO_CLASS == 'Dataset': 689 self.clearDatasetContents(None, client_base, obj)
690
691 - def setDatasetContents (self, r, client_base, ds, data):
692 """ 693 Adds data to the json cache using 'contents' as context 694 695 @param r: http request - not used 696 @param client_base: server_id for cache key 697 @param ds: ObjectWrapper for cache key 698 @param data: Data to cache 699 @rtype: True 700 """ 701 702 k = self._jsonKey(r, client_base, ds, 'contents') 703 self._cache_set(self._json_cache, k, data) 704 return True
705
706 - def getDatasetContents (self, r, client_base, ds):
707 """ 708 Gets data from the json cache using 'contents' as context 709 710 @param r: http request - not used 711 @param client_base: server_id for cache key 712 @param ds: ObjectWrapper for cache key 713 @rtype: String or None 714 """ 715 716 k = self._jsonKey(r, client_base, ds, 'contents') 717 r = self._json_cache.get(k) 718 if r is None: 719 logger.debug(' fail: %s' % k) 720 else: 721 logger.debug('cached: %s' % k) 722 return r
723
724 - def clearDatasetContents (self, r, client_base, ds):
725 """ 726 Clears data from the json cache using 'contents' as context 727 728 @param r: http request - not used 729 @param client_base: server_id for cache key 730 @param ds: ObjectWrapper for cache key 731 @rtype: True 732 """ 733 734 k = self._jsonKey(r, client_base, ds, 'contents') 735 self._cache_clear(self._json_cache, k) 736 return True
737 738 webgateway_cache = WebGatewayCache(FileCache) 739
740 -class AutoLockFile (file):
741 """ Class extends file to facilitate creation and deletion of lock file. """ 742
743 - def __init__ (self, fn, mode):
744 """ creates a '.lock' file with the spicified file name and mode """ 745 super(AutoLockFile, self).__init__(fn, mode) 746 self._lock = os.path.join(os.path.dirname(fn), '.lock') 747 file(self._lock, 'a').close()
748
749 - def __del__ (self):
750 """ tries to delete the lock file """ 751 try: 752 os.remove(self._lock) 753 except: 754 pass
755
756 - def close (self):
757 """ tries to delete the lock file and close the file """ 758 try: 759 os.remove(self._lock) 760 except: 761 pass 762 super(AutoLockFile, self).close()
763
764 -class WebGatewayTempFile (object):
765 """ 766 Class for handling creation of temporary files 767 """ 768
769 - def __init__ (self, tdir=TMPROOT):
770 """ Initialises class, setting the directory to be used for temp files. """ 771 self._dir = tdir 772 if tdir and not os.path.exists(self._dir): 773 self._createdir()
774
775 - def _createdir(self):
776 """ Tries to create the directories required for the temp file base dir """ 777 try: 778 os.makedirs(self._dir) 779 except OSError: #pragma: nocover 780 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
781
782 - def _cleanup (self):
783 """ Tries to delete all the temp files that have expired their cache timeout. """ 784 now = time.time() 785 for f in os.listdir(self._dir): 786 try: 787 ts = os.path.join(self._dir, f, '.timestamp') 788 if os.path.exists(ts): 789 ft = float(file(ts).read()) + TMPDIR_TIME 790 else: 791 ft = float(f) + TMPDIR_TIME 792 if ft < now: 793 shutil.rmtree(os.path.join(self._dir, f), ignore_errors=True) 794 except ValueError: 795 continue
796
797 - def newdir (self, key=None):
798 """ 799 Creates a new directory using key as the dir name, and adds a timestamp file with it's 800 creation time. If key is not specified, use a unique key based on timestamp. 801 802 @param key: The new dir name 803 @return: Tuple of (path to new directory, key used) 804 """ 805 806 if not self._dir: 807 return None, None 808 self._cleanup() 809 stamp = str(time.time()) 810 if key is None: 811 dn = os.path.join(self._dir, stamp) 812 while os.path.exists(dn): 813 stamp = str(time.time()) 814 dn = os.path.join(self._dir, stamp) 815 key = stamp 816 key = key.replace('/','_').decode('utf8').encode('ascii','ignore') 817 dn = os.path.join(self._dir, key) 818 if not os.path.isdir(dn): 819 os.makedirs(dn) 820 file(os.path.join(dn, '.timestamp'), 'w').write(stamp) 821 return dn, key
822
823 - def new (self, name, key=None):
824 """ 825 Creates a new directory if needed, see L{newdir} and checks whether this contains a file 'name'. If not, a 826 file lock is created for this location and returned. 827 828 @param name: Name of file we want to create. 829 @param key: The new dir name 830 @return: Tuple of (abs path to new directory, relative path key/name, L{AutoFileLock} or True if exists) 831 """ 832 833 if not self._dir: 834 return None, None, None 835 dn, stamp = self.newdir(key) 836 name = name.replace('/','_').decode('utf8').encode('ascii', 'ignore') 837 fn = os.path.join(dn, name) 838 rn = os.path.join(stamp, name) 839 lf = os.path.join(dn, '.lock') 840 cnt = 30 841 fsize = 0 842 while os.path.exists(lf) and cnt > 0: 843 time.sleep(1) 844 t = os.stat(fn)[stat.ST_SIZE] 845 if (t == fsize): 846 cnt -= 1 847 logger.debug('countdown %d' % cnt) 848 else: 849 fsize = t 850 cnt = 30 851 if cnt == 0: 852 return None, None, None 853 if os.path.exists(fn): 854 return fn, rn, True 855 return fn, rn, AutoLockFile(fn, 'wb')
856 857 webgateway_tempfile = WebGatewayTempFile() 858