Package omeroweb :: Package webgateway :: Module webgateway_cache
[hide private]
[frames] | no frames]

Source Code for Module omeroweb.webgateway.webgateway_cache

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # 
  4  # webgateway/webgateway_cache - web cache handler for webgateway 
  5  #  
  6  # Copyright (c) 2008, 2009 Glencoe Software, Inc. All rights reserved. 
  7  #  
  8  # This software is distributed under the terms described by the LICENCE file 
  9  # you can find at the root of the distribution bundle, which states you are 
 10  # free to use it only for non commercial purposes. 
 11  # If the file is missing please request a copy by contacting 
 12  # jason@glencoesoftware.com. 
 13  # 
 14  # Author: Carlos Neves <carlos(at)glencoesoftware.com> 
 15   
 16  from django.conf import settings 
 17  import omero 
 18  import logging 
 19  from random import random 
 20  import datetime 
 21  from types import StringTypes 
 22   
 23  logger = logging.getLogger(__name__) 
 24   
 25  import struct, time, os, re, shutil, stat 
 26  size_of_double = len(struct.pack('d',0)) 
 27  #string_type = type('') 
 28   
 29  CACHE=getattr(settings, 'WEBGATEWAY_CACHE', None) 
 30  TMPROOT=getattr(settings, 'WEBGATEWAY_TMPROOT', None) 
 31  THUMB_CACHE_TIME = 3600 # 1 hour 
 32  THUMB_CACHE_SIZE = 20*1024 # KB == 20MB 
 33  IMG_CACHE_TIME= 3600 # 1 hour 
 34  IMG_CACHE_SIZE = 512*1024 # KB == 512MB 
 35  JSON_CACHE_TIME= 3600 # 1 hour 
 36  JSON_CACHE_SIZE = 1*1024 # KB == 1MB 
 37  TMPDIR_TIME = 3600 * 12 # 12 hours 
 38   
39 -class CacheBase (object): #pragma: nocover
40 """ 41 Caching base class - extended by L{FileCache} for file-based caching. 42 Methods of this base class return None or False providing a no-caching implementation if needed 43 """ 44
45 - def __init__ (self):
46 """ not implemented """ 47 pass
48
49 - def get (self, k):
50 return None
51
52 - def set (self, k, v, t=0, invalidateGroup=None):
53 return False
54
55 - def delete (self, k):
56 return False
57
58 - def wipe (self):
59 return False
60
61 -class FileCache(CacheBase):
62 """ 63 Implements file-based caching within the directory specified in constructor. 64 """ 65 _purge_holdoff = 4 66
67 - def __init__(self, dir, timeout=60, max_entries=0, max_size=0):
68 """ 69 Initialises the class. 70 71 @param dir: Path to directory to place cached files. 72 @param timeout: Cache timeout in secs 73 @param max_entries: If specified, limits number of items to cache 74 @param max_size: Maxium size of cache in KB 75 """ 76 77 super(FileCache, self).__init__() 78 self._dir = dir 79 self._max_entries = max_entries 80 self._max_size = max_size 81 self._last_purge = 0 82 self._default_timeout=timeout 83 if not os.path.exists(self._dir): 84 self._createdir()
85 #
86 - def add(self, key, value, timeout=None, invalidateGroup=None):
87 """ 88 Adds data to cache, returning False if already cached. Otherwise delegating to L{set} 89 90 @param key: Unique key for cache 91 @param value: Value to cache - must be String 92 @param timeout: Optional timeout - otherwise use default 93 @param invalidateGroup: Not used? 94 """ 95 96 if self.has_key(key): 97 return False 98 99 self.set(key, value, timeout, invalidateGroup=invalidateGroup) 100 return True
101
102 - def get(self, key, default=None):
103 """ 104 Gets data from cache 105 106 @param key: cache key 107 @param default: default value to return 108 @return: cache data or default if timout has passed 109 """ 110 fname = self._key_to_file(key) 111 try: 112 f = open(fname, 'rb') 113 if not self._check_entry(f): 114 f.close() 115 self._delete(fname) 116 else: 117 return f.read() 118 except (IOError, OSError, EOFError, struct.error): 119 pass 120 return default
121
122 - def set(self, key, value, timeout=None, invalidateGroup=None):
123 """ 124 Adds data to cache, overwriting if already cached. 125 126 @param key: Unique key for cache 127 @param value: Value to cache - must be String 128 @param timeout: Optional timeout - otherwise use default 129 @param invalidateGroup: Not used? 130 """ 131 132 if not isinstance(value, StringTypes): 133 raise ValueError("%s not a string, can't cache" % type(value)) 134 fname = self._key_to_file(key) 135 dirname = os.path.dirname(fname) 136 137 if timeout is None: 138 timeout = self._default_timeout 139 140 if self._full(): 141 # Maybe we already have this one cached, and we need the space 142 try: 143 self._delete(fname) 144 except OSError: 145 pass 146 if self._full(): 147 return 148 149 try: 150 if not os.path.exists(dirname): 151 os.makedirs(dirname) 152 153 f = open(fname, 'wb') 154 if timeout > 0: 155 exp = time.time() + timeout + (timeout / 5 * random()) 156 else: 157 exp = 0 158 f.write(struct.pack('d', exp)) 159 f.write(value) 160 f.close() 161 except (IOError, OSError): #pragma: nocover 162 pass
163
164 - def delete(self, key):
165 """ 166 Attempt to delete the cache data referenced by key 167 @param key: Cache key 168 """ 169 170 try: 171 self._delete(self._key_to_file(key)) 172 except (IOError, OSError): #pragma: nocover 173 pass
174
175 - def _delete(self, fname):
176 """ 177 Tries to delete the data at the specified absolute file path 178 179 @param fname: File name of data to delete 180 """ 181 182 logger.debug('requested delete for "%s"' % fname) 183 if os.path.isdir(fname): 184 shutil.rmtree(fname, ignore_errors=True) 185 else: 186 os.remove(fname) 187 try: 188 # Remove the parent subdirs if they're empty 189 dirname = os.path.dirname(fname) 190 while dirname != self._dir: 191 os.rmdir(dirname) 192 dirname = os.path.dirname(fname) 193 except (IOError, OSError): 194 pass
195
196 - def wipe (self):
197 """ Deletes everything in the cache """ 198 199 shutil.rmtree(self._dir) 200 self._createdir() 201 return True
202
203 - def _check_entry (self, fname):
204 """ 205 Verifies if a specific cache entry (provided as absolute file path) is expired. 206 If expired, it gets deleted and method returns false. 207 If not expired, returns True. 208 209 If fname is a file object, fpos will advance size_of_double bytes. 210 211 @param fname: File path or file object 212 @rtype Boolean 213 @return True if entry is valid, False if expired 214 """ 215 try: 216 if isinstance(fname, StringTypes): 217 f = open(fname, 'rb') 218 exp = struct.unpack('d',f.read(size_of_double))[0] 219 else: 220 f = None 221 exp = struct.unpack('d',fname.read(size_of_double))[0] 222 if self._default_timeout > 0 and exp > 0: 223 now = time.time() 224 if exp < now: 225 if f is not None: 226 f.close() 227 self._delete(fname) 228 return False 229 else: 230 return True 231 return True 232 except (IOError, OSError, EOFError, struct.error): #pragma: nocover 233 return False
234
235 - def has_key(self, key):
236 """ 237 Returns true if the cache has the specified key 238 @param key: Key to look for. 239 @rtype: Boolean 240 """ 241 fname = self._key_to_file(key) 242 return self._check_entry(fname)
243
244 - def _du (self):
245 """ 246 Disk Usage count on the filesystem the cache is based at 247 248 @rtype: int 249 @return: the current usage, in KB 250 """ 251 return int(os.popen('du -sk %s' % os.path.join(os.getcwd(),self._dir)).read().split('\t')[0].strip())
252
253 - def _full(self, _on_retry=False):
254 """ 255 Checks whether the cache is full, either because we have exceeded max number of entries or 256 the cache space is full. 257 258 @param _on_retry: Flag allows calling this method again after purge() without recursion 259 @return: True if cache is full 260 @rtype: Boolean 261 """ 262 263 # Check nr of entries 264 if self._max_entries: 265 try: 266 x = int(os.popen('find %s -type f | wc -l' % self._dir).read().strip()) 267 if x >= self._max_entries: 268 if not _on_retry: 269 self._purge() 270 return self._full(True) 271 logger.warn('caching limits reached on %s: max entries %d' % (self._dir, self._max_entries)) 272 return True 273 except ValueError: #pragma: nocover 274 logger.error('Counting cache entries failed') 275 # Check for space usage 276 if self._max_size: 277 try: 278 x = self._du() 279 if x >= self._max_size: 280 if not _on_retry: 281 self._purge() 282 return self._full(True) 283 logger.warn('caching limits reached on %s: max size %d' % (self._dir, self._max_size)) 284 return True 285 except ValueError: #pragma: nocover 286 logger.error('Counting cache size failed') 287 return False
288
289 - def _purge (self):
290 """ 291 Iterate the whole cache structure searching and cleaning expired entries. 292 this method may be expensive, so only call it when really necessary. 293 """ 294 now = time.time() 295 if now-self._last_purge < self._purge_holdoff: 296 return 297 self._last_purge = now 298 299 logger.debug('entering purge') 300 count = 0 301 for p,_,files in os.walk(self._dir): 302 for f in files: 303 if not self._check_entry(os.path.join(p, f)): 304 count += 1 305 logger.debug('purge finished, removed %d files' % count)
306
307 - def _createdir(self):
308 """ 309 Creates a directory for the root dir of the cache. 310 """ 311 try: 312 os.makedirs(self._dir) 313 except OSError: #pragma: nocover 314 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
315
316 - def _key_to_file(self, key):
317 """ 318 Uses the key to construct an absolute path to the cache data. 319 @param key: Cache key 320 @return: Path 321 @rtype: String 322 """ 323 324 if key.find('..') > 0 or key.startswith('/'): 325 raise ValueError('Invalid value for cache key: "%s"' % key) 326 return os.path.join(self._dir, key)
327
328 - def _get_num_entries(self):
329 """ 330 Returns the number of files in the cache 331 @rtype: int 332 """ 333 count = 0 334 for _,_,files in os.walk(self._dir): 335 count += len(files) 336 return count
337 _num_entries = property(_get_num_entries)
338 339 FN_REGEX = re.compile('[#$,|]')
340 -class WebGatewayCache (object):
341 """ 342 Caching class for webgateway. 343 """ 344
345 - def __init__ (self, backend=None, basedir=CACHE):
346 """ 347 Initialises cache 348 349 @param backend: The cache class to use for caching. E.g. L{FileCache} 350 @param basedir: The base location for all caches. Sub-dirs created for json/ img/ thumb/ 351 """ 352 353 self._basedir = basedir 354 self._lastlock = None 355 if backend is None or basedir is None: 356 self._json_cache = CacheBase() 357 self._img_cache = CacheBase() 358 self._thumb_cache = CacheBase() 359 else: 360 self._json_cache = backend(dir=os.path.join(basedir,'json'), 361 timeout=JSON_CACHE_TIME, max_entries=0, max_size=JSON_CACHE_SIZE) 362 self._img_cache = backend(dir=os.path.join(basedir,'img'), 363 timeout=IMG_CACHE_TIME, max_entries=0, max_size=IMG_CACHE_SIZE) 364 self._thumb_cache = backend(dir=os.path.join(basedir,'thumb'), 365 timeout=THUMB_CACHE_TIME, max_entries=0, max_size=THUMB_CACHE_SIZE)
366
367 - def _updateCacheSettings (self, cache, timeout=None, max_entries=None, max_size=None):
368 """ 369 Updates the timeout, max_entries and max_size (if specified) for the given cache 370 371 @param cache: Cache or caches to update. 372 @type cache: L{CacheBase} or list of caches 373 """ 374 375 if isinstance(cache, CacheBase): 376 cache = (cache,) 377 for c in cache: 378 if timeout is not None: 379 c._default_timeout = timeout 380 if max_entries is not None: 381 c._max_entries = max_entries 382 if max_size is not None: 383 c._max_size = max_size
384
385 - def __del__ (self):
386 """ 387 Tries to remove the lock on this cache. 388 """ 389 if self._lastlock: 390 try: 391 logger.debug('removing cache lock file on __del__') 392 os.remove(self._lastlock) 393 except: 394 pass 395 self._lastlock = None
396
397 - def tryLock (self):
398 """ 399 simple lock mechanisn to avoid multiple processes on the same cache to 400 step on each other's toes. 401 402 @rtype: boolean 403 @return: True if we created a lockfile or already had it. False otherwise. 404 """ 405 lockfile = os.path.join(self._basedir, '%s_lock' % datetime.datetime.now().strftime('%Y%m%d_%H%M')) 406 if self._lastlock: 407 if lockfile == self._lastlock: 408 return True 409 try: 410 os.remove(self._lastlock) 411 except: 412 pass 413 self._lastlock = None 414 try: 415 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL) 416 os.close(fd) 417 self._lastlock = lockfile 418 return True 419 except OSError: 420 return False
421
422 - def handleEvent (self, client_base, e):
423 """ 424 Handle one event from blitz.onEventLogs. 425 426 Meant to be overridden, this implementation just logs. 427 428 @param client_base: TODO: docs! 429 @param e: 430 """ 431 logger.debug('## %s#%i %s user #%i group #%i(%i)' % (e.entityType.val, 432 e.entityId.val, 433 e.action.val, 434 e.details.owner.id.val, 435 e.details.group.id.val, 436 e.event.id.val))
437
438 - def eventListener (self, client_base, events):
439 """ 440 handle events coming our way from blitz.onEventLogs. 441 442 Because all processes will be listening to the same events, we use a simple file 443 lock mechanism to make sure the first process to get the event will be the one 444 handling things from then on. 445 446 @param client_base: TODO: docs! 447 @param events: 448 """ 449 for e in events: 450 if self.tryLock(): 451 self.handleEvent(client_base, e) 452 else: 453 logger.debug("## ! ignoring event %s" % str(e.event.id.val))
454
455 - def clear (self):
456 """ 457 Clears all the caches. 458 """ 459 self._json_cache.wipe() 460 self._img_cache.wipe() 461 self._thumb_cache.wipe()
462
463 - def _cache_set (self, cache, key, obj):
464 """ Calls cache.set(key, obj) """ 465 466 logger.debug(' set: %s' % key) 467 cache.set(key, obj)
468
469 - def _cache_clear (self, cache, key):
470 """ Calls cache.delete(key) """ 471 472 logger.debug(' clear: %s' % key) 473 cache.delete(key)
474
475 - def invalidateObject (self, client_base, user_id, obj):
476 """ 477 Invalidates all caches for this particular object 478 479 @param client_base: The server_id 480 @param user_id: OMERO user ID to partition caching upon 481 @param obj: The object wrapper. E.g. L{omero.gateway.ImageWrapper} 482 """ 483 484 if obj.OMERO_CLASS == 'Image': 485 self.clearImage(None, client_base, user_id, obj) 486 else: 487 logger.debug('unhandled object type: %s' % obj.OMERO_CLASS) 488 self.clearJson(client_base, obj)
489 490 ## 491 # Thumb 492
493 - def _thumbKey (self, r, client_base, user_id, iid, size):
494 """ 495 Generates a string key for caching the thumbnail, based on the above parameters 496 497 @param r: not used 498 @param client_base: server-id, forms stem of the key 499 @param user_id: OMERO user ID to partition caching upon 500 @param iid: image ID 501 @param size: size of the thumbnail - tuple. E.g. (100,) 502 """ 503 pre = str(iid)[:-4] 504 if len(pre) == 0: 505 pre = '0' 506 if size is not None and len(size): 507 return 'thumb_user_%s/%s/%s/%s/%s' % (client_base, pre, str(iid), user_id, 'x'.join([str(x) for x in size])) 508 else: 509 return 'thumb_user_%s/%s/%s/%s' % (client_base, pre, str(iid), user_id)
510
511 - def setThumb (self, r, client_base, user_id, iid, obj, size=()):
512 """ 513 Puts thumbnail into cache. 514 515 @param r: for cache key - Not used? 516 @param client_base: server_id for cache key 517 @param user_id: OMERO user ID to partition caching upon 518 @param iid: image ID for cache key 519 @param obj: Data to cache 520 @param size: Size used for cache key. Tuple 521 """ 522 523 k = self._thumbKey(r, client_base, user_id, iid, size) 524 self._cache_set(self._thumb_cache, k, obj) 525 return True
526
527 - def getThumb (self, r, client_base, user_id, iid, size=()):
528 """ 529 Gets thumbnail from cache. 530 531 @param r: for cache key - Not used? 532 @param client_base: server_id for cache key 533 @param user_id: OMERO user ID to partition caching upon 534 @param iid: image ID for cache key 535 @param size: Size used for cache key. Tuple 536 @return: Cached data or None 537 @rtype: String 538 """ 539 540 k = self._thumbKey(r, client_base, user_id, iid, size) 541 r = self._thumb_cache.get(k) 542 if r is None: 543 logger.debug(' fail: %s' % k) 544 else: 545 logger.debug('cached: %s' % k) 546 return r
547
548 - def clearThumb (self, r, client_base, user_id, iid, size=None):
549 """ 550 Clears thumbnail from cache. 551 552 @param r: for cache key - Not used? 553 @param client_base: server_id for cache key 554 @param user_id: OMERO user ID to partition caching upon 555 @param iid: image ID for cache key 556 @param size: Size used for cache key. Tuple 557 @return: True 558 """ 559 k = self._thumbKey(r, client_base, user_id, iid, size) 560 self._cache_clear(self._thumb_cache, k) 561 return True
562 563 ## 564 # Image 565
566 - def _imageKey (self, r, client_base, img, z=0, t=0):
567 """ 568 Returns a key for caching the Image, based on parameters above, including rendering settings 569 specified in the http request. 570 571 @param r: http request - get rendering params 'c', 'm', 'p' 572 @param client_base: server_id for cache key 573 @param img: L{omero.gateway.ImageWrapper} for ID 574 @param obj: Data to cache 575 @param size: Size used for cache key. Tuple 576 """ 577 578 iid = img.getId() 579 pre = str(iid)[:-4] 580 if len(pre) == 0: 581 pre = '0' 582 if r: 583 r = r.REQUEST 584 c = FN_REGEX.sub('-',r.get('c', '')) 585 m = r.get('m', '') 586 p = r.get('p', '') 587 if p and not isinstance(omero.gateway.ImageWrapper.PROJECTIONS.get(p, -1), 588 omero.constants.projection.ProjectionType): #pragma: nocover 589 p = '' 590 q = r.get('q', '') 591 region = r.get('region', '') 592 tile = r.get('tile', '') 593 rv = 'img_%s/%s/%s/%%s-c%s-m%s-q%s-r%s-t%s' % (client_base, pre, str(iid), c, m, q, region, tile) 594 if p: 595 return rv % ('%s-%s' % (p, str(t))) 596 else: 597 return rv % ('%sx%s' % (str(z), str(t))) 598 else: 599 return 'img_%s/%s/%s' % (client_base, pre, str(iid))
600
601 - def setImage (self, r, client_base, img, z, t, obj, ctx=''):
602 """ 603 Puts image data into cache. 604 605 @param r: http request for cache key 606 @param client_base: server_id for cache key 607 @param img: ImageWrapper for cache key 608 @param z: Z index for cache key 609 @param t: T index for cache key 610 @param obj: Data to cache 611 @param ctx: Additional string for cache key 612 """ 613 614 k = self._imageKey(r, client_base, img, z, t) + ctx 615 self._cache_set(self._img_cache, k, obj) 616 return True
617
618 - def getImage (self, r, client_base, img, z, t, ctx=''):
619 """ 620 Gets image data from cache. 621 622 @param r: http request for cache key 623 @param client_base: server_id for cache key 624 @param img: ImageWrapper for cache key 625 @param z: Z index for cache key 626 @param t: T index for cache key 627 @param ctx: Additional string for cache key 628 @return: Image data 629 @rtype: String 630 """ 631 k = self._imageKey(r, client_base, img, z, t) + ctx 632 r = self._img_cache.get(k) 633 if r is None: 634 logger.debug(' fail: %s' % k) 635 else: 636 logger.debug('cached: %s' % k) 637 return r
638
639 - def clearImage (self, r, client_base, user_id, img, skipJson=False):
640 """ 641 Clears image data from cache using default rendering settings (r=None) T and Z indexes ( = 0). 642 TODO: Doesn't clear any data stored WITH r, t, or z specified in cache key? 643 Also clears thumbnail (but not thumbs with size specified) and json data for this image. 644 645 @param r: http request for cache key 646 @param client_base: server_id for cache key 647 @param user_id: OMERO user ID to partition caching upon 648 @param img: ImageWrapper for cache key 649 @param obj: Data to cache 650 @param rtype: True 651 """ 652 653 k = self._imageKey(None, client_base, img) 654 self._cache_clear(self._img_cache, k) 655 # do the thumb too 656 self.clearThumb(r, client_base, user_id, img.getId()) 657 # and json data 658 if not skipJson: 659 self.clearJson(client_base, img) 660 return True
661
662 - def setSplitChannelImage (self, r, client_base, img, z, t, obj):
663 """ Calls L{setImage} with '-sc' context """ 664 return self.setImage(r, client_base, img, z, t, obj, '-sc')
665
666 - def getSplitChannelImage (self, r, client_base, img, z, t):
667 """ 668 Calls L{getImage} with '-sc' context 669 @rtype: String 670 """ 671 return self.getImage(r, client_base, img, z, t, '-sc')
672
673 - def setOmeTiffImage (self, r, client_base, img, obj):
674 """ Calls L{setImage} with '-ometiff' context """ 675 return self.setImage(r, client_base, img, 0, 0, obj, '-ometiff')
676
677 - def getOmeTiffImage (self, r, client_base, img):
678 """ 679 Calls L{getImage} with '-ometiff' context 680 @rtype: String 681 """ 682 return self.getImage(r, client_base, img, 0, 0, '-ometiff')
683 684 ## 685 # hierarchies (json) 686
687 - def _jsonKey (self, r, client_base, obj, ctx=''):
688 """ 689 Creates a cache key for storing json data based on params above. 690 691 @param r: http request - not used 692 @param client_base: server_id 693 @param obj: ObjectWrapper 694 @param ctx: Additional string for cache key 695 @return: Cache key 696 @rtype: String 697 """ 698 699 if obj: 700 return 'json_%s/%s_%s/%s' % (client_base, obj.OMERO_CLASS, obj.id, ctx) 701 else: 702 return 'json_%s/single/%s' % (client_base, ctx)
703
704 - def getJson (self, r, client_base, obj, ctx=''):
705 """ 706 Gets data from the json cache 707 708 @param r: http request - not used 709 @param client_base: server_id for cache key 710 @param obj: ObjectWrapper for cache key 711 @param ctx: context string used for cache key 712 @rtype: String or None 713 """ 714 k = self._jsonKey(r, client_base, obj, ctx) 715 r = self._json_cache.get(k) 716 if r is None: 717 logger.debug(' fail: %s' % k) 718 else: 719 logger.debug('cached: %s' % k) 720 return r
721
722 - def setJson (self, r, client_base, obj, data, ctx=''):
723 """ 724 Adds data to the json cache 725 726 @param r: http request - not used 727 @param client_base: server_id for cache key 728 @param obj: ObjectWrapper for cache key 729 @param data: Data to cache 730 @param ctx: context string used for cache key 731 @rtype: True 732 """ 733 k = self._jsonKey(r, client_base, obj, ctx) 734 self._cache_set(self._json_cache, k, data) 735 return True
736
737 - def clearJson (self, client_base, obj, ctx=''):
738 """ 739 TODO: document 740 WAS: Only handles Dataset obj, calling L{clearDatasetContents} 741 """ 742 k = self._jsonKey(None, client_base, obj, ctx) 743 self._cache_clear(self._json_cache, k) 744 return True
745 #logger.debug('clearjson') 746 #if obj.OMERO_CLASS == 'Dataset': 747 # self.clearDatasetContents(None, client_base, obj) 748
749 - def setDatasetContents (self, r, client_base, ds, data):
750 """ 751 Adds data to the json cache using 'contents' as context 752 753 @param r: http request - not used 754 @param client_base: server_id for cache key 755 @param ds: ObjectWrapper for cache key 756 @param data: Data to cache 757 @rtype: True 758 """ 759 return self.setJson(r, client_base, ds, data, 'contents')
760
761 - def getDatasetContents (self, r, client_base, ds):
762 """ 763 Gets data from the json cache using 'contents' as context 764 765 @param r: http request - not used 766 @param client_base: server_id for cache key 767 @param ds: ObjectWrapper for cache key 768 @rtype: String or None 769 """ 770 return self.getJson(r, client_base, ds, 'contents')
771
772 - def clearDatasetContents (self, r, client_base, ds):
773 """ 774 Clears data from the json cache using 'contents' as context 775 776 @param r: http request - not used 777 @param client_base: server_id for cache key 778 @param ds: ObjectWrapper for cache key 779 @rtype: True 780 """ 781 782 k = self._jsonKey(r, client_base, ds, 'contents') 783 self._cache_clear(self._json_cache, k) 784 return True
785 786 webgateway_cache = WebGatewayCache(FileCache) 787
788 -class AutoLockFile (file):
789 """ Class extends file to facilitate creation and deletion of lock file. """ 790
791 - def __init__ (self, fn, mode):
792 """ creates a '.lock' file with the spicified file name and mode """ 793 super(AutoLockFile, self).__init__(fn, mode) 794 self._lock = os.path.join(os.path.dirname(fn), '.lock') 795 file(self._lock, 'a').close()
796
797 - def __del__ (self):
798 """ tries to delete the lock file """ 799 try: 800 os.remove(self._lock) 801 except: 802 pass
803
804 - def close (self):
805 """ tries to delete the lock file and close the file """ 806 try: 807 os.remove(self._lock) 808 except: 809 pass 810 super(AutoLockFile, self).close()
811
812 -class WebGatewayTempFile (object):
813 """ 814 Class for handling creation of temporary files 815 """ 816
817 - def __init__ (self, tdir=TMPROOT):
818 """ Initialises class, setting the directory to be used for temp files. """ 819 self._dir = tdir 820 if tdir and not os.path.exists(self._dir): 821 self._createdir()
822
823 - def _createdir(self):
824 """ Tries to create the directories required for the temp file base dir """ 825 try: 826 os.makedirs(self._dir) 827 except OSError: #pragma: nocover 828 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
829
830 - def _cleanup (self):
831 """ Tries to delete all the temp files that have expired their cache timeout. """ 832 now = time.time() 833 for f in os.listdir(self._dir): 834 try: 835 ts = os.path.join(self._dir, f, '.timestamp') 836 if os.path.exists(ts): 837 ft = float(file(ts).read()) + TMPDIR_TIME 838 else: 839 ft = float(f) + TMPDIR_TIME 840 if ft < now: 841 shutil.rmtree(os.path.join(self._dir, f), ignore_errors=True) 842 except ValueError: 843 continue
844
845 - def newdir (self, key=None):
846 """ 847 Creates a new directory using key as the dir name, and adds a timestamp file with it's 848 creation time. If key is not specified, use a unique key based on timestamp. 849 850 @param key: The new dir name 851 @return: Tuple of (path to new directory, key used) 852 """ 853 854 if not self._dir: 855 return None, None 856 self._cleanup() 857 stamp = str(time.time()) 858 if key is None: 859 dn = os.path.join(self._dir, stamp) 860 while os.path.exists(dn): 861 stamp = str(time.time()) 862 dn = os.path.join(self._dir, stamp) 863 key = stamp 864 key = key.replace('/','_').decode('utf8').encode('ascii','ignore') 865 dn = os.path.join(self._dir, key) 866 if not os.path.isdir(dn): 867 os.makedirs(dn) 868 file(os.path.join(dn, '.timestamp'), 'w').write(stamp) 869 return dn, key
870
871 - def abort (self, fn):
872 logger.debug(fn) 873 logger.debug(os.path.dirname(fn)) 874 logger.debug(self._dir) 875 if fn.startswith(self._dir): 876 shutil.rmtree(os.path.dirname(fn), ignore_errors=True)
877
878 - def new (self, name, key=None):
879 """ 880 Creates a new directory if needed, see L{newdir} and checks whether this contains a file 'name'. If not, a 881 file lock is created for this location and returned. 882 883 @param name: Name of file we want to create. 884 @param key: The new dir name 885 @return: Tuple of (abs path to new directory, relative path key/name, L{AutoFileLock} or True if exists) 886 """ 887 888 if not self._dir: 889 return None, None, None 890 dn, stamp = self.newdir(key) 891 name = name.replace('/','_').replace('#','_').decode('utf8').encode('ascii', 'ignore') 892 if len(name)>255: 893 # Try to be smart about trimming and keep up to two levels of extension (ex: .ome.tiff) 894 # We do limit the extension to 16 chars just to keep things sane 895 fname, fext = os.path.splitext(name) 896 if fext: 897 if len(fext) <= 16: 898 fname, fext2 = os.path.splitext(fname) 899 if len(fext+fext2) <= 16: 900 fext = fext2 + fext 901 else: 902 fname += fext2 903 else: 904 fname = name 905 fext = '' 906 name = fname[:-len(name)+255] + fext 907 fn = os.path.join(dn, name) 908 rn = os.path.join(stamp, name) 909 lf = os.path.join(dn, '.lock') 910 cnt = 30 911 fsize = 0 912 while os.path.exists(lf) and cnt > 0: 913 time.sleep(1) 914 t = os.stat(fn)[stat.ST_SIZE] 915 if (t == fsize): 916 cnt -= 1 917 logger.debug('countdown %d' % cnt) 918 else: 919 fsize = t 920 cnt = 30 921 if cnt == 0: 922 return None, None, None 923 if os.path.exists(fn): 924 return fn, rn, True 925 return fn, rn, AutoLockFile(fn, 'wb')
926 927 webgateway_tempfile = WebGatewayTempFile() 928