1
2
3
4
5
6
7
8
9
10
11
12
13
14 from django.conf import settings
15 import omero
16 import logging
17 from random import random
18 import datetime
19
20
21 logger = logging.getLogger('cache')
22
23 import struct, time, os, re, shutil, stat
24 size_of_double = len(struct.pack('d',0))
25 string_type = type('')
26
27 CACHE=getattr(settings, 'WEBGATEWAY_CACHE', None)
28 TMPROOT=getattr(settings, 'WEBGATEWAY_TMPROOT', None)
29 THUMB_CACHE_TIME = 3600
30 THUMB_CACHE_SIZE = 20*1024
31 IMG_CACHE_TIME= 3600
32 IMG_CACHE_SIZE = 512*1024
33 JSON_CACHE_TIME= 3600
34 JSON_CACHE_SIZE = 1*1024
35 TMPDIR_TIME = 3600 * 12
36
38 """
39 Caching base class - extended by L{FileCache} for file-based caching.
40 Methods of this base class return None or False providing a no-caching implementation if needed
41 """
42
44 """ not implemented """
45 pass
46
49
50 - def set (self, k, v, t=0, invalidateGroup=None):
52
55
58
60 """
61 Implements file-based caching within the directory specified in constructor.
62 """
63 _purge_holdoff = 4
64
65 - def __init__(self, dir, timeout=60, max_entries=0, max_size=0):
66 """
67 Initialises the class.
68
69 @param dir: Path to directory to place cached files.
70 @param timeout: Cache timeout in secs
71 @param max_entries: If specified, limits number of items to cache
72 @param max_size: Maxium size of cache in KB
73 """
74
75 super(FileCache, self).__init__()
76 self._dir = dir
77 self._max_entries = max_entries
78 self._max_size = max_size
79 self._last_purge = 0
80 self._default_timeout=timeout
81 if not os.path.exists(self._dir):
82 self._createdir()
83
84 - def add(self, key, value, timeout=None, invalidateGroup=None):
85 """
86 Adds data to cache, returning False if already cached. Otherwise delegating to L{set}
87
88 @param key: Unique key for cache
89 @param value: Value to cache - must be String
90 @param timeout: Optional timeout - otherwise use default
91 @param invalidateGroup: Not used?
92 """
93
94 if self.has_key(key):
95 return False
96
97 self.set(key, value, timeout, invalidateGroup=invalidateGroup)
98 return True
99
100 - def get(self, key, default=None):
101 """
102 Gets data from cache
103
104 @param key: cache key
105 @param default: default value to return
106 @return: cache data or default if timout has passed
107 """
108 fname = self._key_to_file(key)
109 try:
110 f = open(fname, 'rb')
111 exp = struct.unpack('d',f.read(size_of_double))[0]
112 now = time.time()
113 if exp < now:
114 f.close()
115 self._delete(fname)
116 else:
117 return f.read()
118 except (IOError, OSError, EOFError, struct.error):
119 pass
120 return default
121
122 - def set(self, key, value, timeout=None, invalidateGroup=None):
123 """
124 Adds data to cache, overwriting if already cached.
125
126 @param key: Unique key for cache
127 @param value: Value to cache - must be String
128 @param timeout: Optional timeout - otherwise use default
129 @param invalidateGroup: Not used?
130 """
131
132 if type(value) != string_type:
133 raise ValueError("%s not a string, can't cache" % type(value))
134 fname = self._key_to_file(key)
135 dirname = os.path.dirname(fname)
136
137 if timeout is None:
138 timeout = self._default_timeout
139
140 if self._full():
141
142 try:
143 self._delete(fname)
144 except OSError:
145 pass
146 if self._full():
147 return
148
149 try:
150 if not os.path.exists(dirname):
151 os.makedirs(dirname)
152
153 f = open(fname, 'wb')
154 exp = time.time() + timeout + (timeout / 5 * random())
155 f.write(struct.pack('d', exp))
156 f.write(value)
157 except (IOError, OSError):
158 pass
159
161 """
162 Attempt to delete the cache data referenced by key
163 @param key: Cache key
164 """
165
166 try:
167 self._delete(self._key_to_file(key))
168 except (IOError, OSError):
169 pass
170
172 """
173 Tries to delete the data at the specified absolute file path
174
175 @param fname: File name of data to delete
176 """
177
178 logger.debug('requested delete for "%s"' % fname)
179 if os.path.isdir(fname):
180 shutil.rmtree(fname, ignore_errors=True)
181 else:
182 os.remove(fname)
183 try:
184
185 dirname = os.path.dirname(fname)
186 while dirname != self._dir:
187 os.rmdir(dirname)
188 dirname = os.path.dirname(fname)
189 except (IOError, OSError):
190 pass
191
193 """ Deletes everything in the cache """
194
195 shutil.rmtree(self._dir)
196 self._createdir()
197 return True
198
199 - def _check_entry (self, fname):
200 """
201 Verifies if a specific cache entry (provided as absolute file path) is expired.
202 If expired, it gets deleted and method returns false.
203 If not expired, returns True.
204
205 @param fname: File path
206 """
207 try:
208 f = open(fname, 'rb')
209 exp = struct.unpack('d',f.read(size_of_double))[0]
210 now = time.time()
211 if exp < now:
212 f.close()
213 self._delete(fname)
214 return False
215 else:
216 return True
217 except (IOError, OSError, EOFError, struct.error):
218 return False
219
221 """
222 Returns true if the cache has the specified key
223 @param key: Key to look for.
224 @rtype: Boolean
225 """
226 fname = self._key_to_file(key)
227 return self._check_entry(fname)
228
230 """
231 Disk Usage count on the filesystem the cache is based at
232
233 @rtype: int
234 @return: the current usage, in KB
235 """
236 return int(os.popen('du -sk %s' % os.path.join(os.getcwd(),self._dir)).read().split('\t')[0].strip())
237
238 - def _full(self, _on_retry=False):
239 """
240 Checks whether the cache is full, either because we have exceeded max number of entries or
241 the cache space is full.
242
243 @param _on_retry: Flag allows calling this method again after purge() without recursion
244 @return: True if cache is full
245 @rtype: Boolean
246 """
247
248
249 if self._max_entries:
250 try:
251 x = int(os.popen('find %s -type f | wc -l' % self._dir).read().strip())
252 if x >= self._max_entries:
253 if not _on_retry:
254 self._purge()
255 return self._full(True)
256 logger.warn('caching limits reached on %s: max entries %d' % (self._dir, self._max_entries))
257 return True
258 except ValueError:
259 logger.error('Counting cache entries failed')
260
261 if self._max_size:
262 try:
263 x = self._du()
264 if x >= self._max_size:
265 if not _on_retry:
266 self._purge()
267 return self._full(True)
268 logger.warn('caching limits reached on %s: max size %d' % (self._dir, self._max_size))
269 return True
270 except ValueError:
271 logger.error('Counting cache size failed')
272 return False
273
275 """
276 Iterate the whole cache structure searching and cleaning expired entries.
277 this method may be expensive, so only call it when really necessary.
278 """
279 now = time.time()
280 if now-self._last_purge < self._purge_holdoff:
281 return
282 self._last_purge = now
283
284 logger.debug('entering purge')
285 count = 0
286 for p,_,files in os.walk(self._dir):
287 for f in files:
288 if not self._check_entry(os.path.join(p, f)):
289 count += 1
290 logger.debug('purge finished, removed %d files' % count)
291
293 """
294 Creates a directory for the root dir of the cache.
295 """
296 try:
297 os.makedirs(self._dir)
298 except OSError:
299 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
300
302 """
303 Uses the key to construct an absolute path to the cache data.
304 @param key: Cache key
305 @return: Path
306 @rtype: String
307 """
308
309 if key.find('..') > 0 or key.startswith('/'):
310 raise ValueError('Invalid value for cache key: "%s"' % key)
311 return os.path.join(self._dir, key)
312
314 """
315 Returns the number of files in the cache
316 @rtype: int
317 """
318 count = 0
319 for _,_,files in os.walk(self._dir):
320 count += len(files)
321 return count
322 _num_entries = property(_get_num_entries)
323
324 FN_REGEX = re.compile('[#$,|]')
326 """
327 Caching class for webgateway.
328 """
329
331 """
332 Initialises cache
333
334 @param backend: The cache class to use for caching. E.g. L{FileCache}
335 @param basedir: The base location for all caches. Sub-dirs created for json/ img/ thumb/
336 """
337
338 self._basedir = basedir
339 self._lastlock = None
340 if backend is None or basedir is None:
341 self._json_cache = CacheBase()
342 self._img_cache = CacheBase()
343 self._thumb_cache = CacheBase()
344 else:
345 self._json_cache = backend(dir=os.path.join(basedir,'json'),
346 timeout=JSON_CACHE_TIME, max_entries=0, max_size=JSON_CACHE_SIZE)
347 self._img_cache = backend(dir=os.path.join(basedir,'img'),
348 timeout=IMG_CACHE_TIME, max_entries=0, max_size=IMG_CACHE_SIZE)
349 self._thumb_cache = backend(dir=os.path.join(basedir,'thumb'),
350 timeout=THUMB_CACHE_TIME, max_entries=0, max_size=THUMB_CACHE_SIZE)
351
353 """
354 Updates the timeout, max_entries and max_size (if specified) for the given cache
355
356 @param cache: Cache or caches to update.
357 @type cache: L{CacheBase} or list of caches
358 """
359
360 if isinstance(cache, CacheBase):
361 cache = (cache,)
362 for c in cache:
363 if timeout is not None:
364 c._default_timeout = timeout
365 if max_entries is not None:
366 c._max_entries = max_entries
367 if max_size is not None:
368 c._max_size = max_size
369
371 """
372 Tries to remove the lock on this cache.
373 """
374 if self._lastlock:
375 try:
376 logger.debug('removing cache lock file on __del__')
377 os.remove(self._lastlock)
378 except:
379 pass
380 self._lastlock = None
381
383 """
384 simple lock mechanisn to avoid multiple processes on the same cache to
385 step on each other's toes.
386
387 @rtype: boolean
388 @return: True if we created a lockfile or already had it. False otherwise.
389 """
390 lockfile = os.path.join(self._basedir, '%s_lock' % datetime.datetime.now().strftime('%Y%m%d_%H%M'))
391 if self._lastlock:
392 if lockfile == self._lastlock:
393 return True
394 try:
395 os.remove(self._lastlock)
396 except:
397 pass
398 self._lastlock = None
399 try:
400 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL)
401 os.close(fd)
402 self._lastlock = lockfile
403 return True
404 except OSError:
405 return False
406
408 """
409 Handle one event from blitz.onEventLogs.
410
411 Meant to be overridden, this implementation just logs.
412
413 @param client_base: TODO: docs!
414 @param e:
415 """
416 logger.debug('## %s#%i %s user #%i group #%i(%i)' % (e.entityType.val,
417 e.entityId.val,
418 e.action.val,
419 e.details.owner.id.val,
420 e.details.group.id.val,
421 e.event.id.val))
422
424 """
425 handle events coming our way from blitz.onEventLogs.
426
427 Because all processes will be listening to the same events, we use a simple file
428 lock mechanism to make sure the first process to get the event will be the one
429 handling things from then on.
430
431 @param client_base: TODO: docs!
432 @param events:
433 """
434 for e in events:
435 if self.tryLock():
436 self.handleEvent(client_base, e)
437 else:
438 logger.debug("## ! ignoring event %s" % str(e.event.id.val))
439
441 """
442 Clears all the caches.
443 """
444 self._json_cache.wipe()
445 self._img_cache.wipe()
446 self._thumb_cache.wipe()
447
453
459
461 """
462 Invalidates all caches for this particular object
463
464 @param client_base: The server_id
465 @param user_id: OMERO user ID to partition caching upon
466 @param obj: The object wrapper. E.g. L{omero.gateway.ImageWrapper}
467 """
468
469 if obj.OMERO_CLASS == 'Image':
470 self.clearImage(None, client_base, user_id, obj)
471 else:
472 logger.debug('unhandled object type: %s' % obj.OMERO_CLASS)
473 self.clearJson(client_base, obj)
474
475
476
477
478 - def _thumbKey (self, r, client_base, user_id, iid, size):
479 """
480 Generates a string key for caching the thumbnail, based on the above parameters
481
482 @param r: not used
483 @param client_base: server-id, forms stem of the key
484 @param user_id: OMERO user ID to partition caching upon
485 @param iid: image ID
486 @param size: size of the thumbnail - tuple. E.g. (100,)
487 """
488
489 if size is not None and len(size):
490 return 'thumb_user_%s/%s/%s/%s' % (client_base, str(iid), user_id, 'x'.join([str(x) for x in size]))
491 else:
492 return 'thumb_user_%s/%s/%s' % (client_base, str(iid), user_id)
493
494 - def setThumb (self, r, client_base, user_id, iid, obj, size=()):
495 """
496 Puts thumbnail into cache.
497
498 @param r: for cache key - Not used?
499 @param client_base: server_id for cache key
500 @param user_id: OMERO user ID to partition caching upon
501 @param iid: image ID for cache key
502 @param obj: Data to cache
503 @param size: Size used for cache key. Tuple
504 """
505
506 k = self._thumbKey(r, client_base, user_id, iid, size)
507 self._cache_set(self._thumb_cache, k, obj)
508 return True
509
510 - def getThumb (self, r, client_base, user_id, iid, size=()):
511 """
512 Gets thumbnail from cache.
513
514 @param r: for cache key - Not used?
515 @param client_base: server_id for cache key
516 @param user_id: OMERO user ID to partition caching upon
517 @param iid: image ID for cache key
518 @param size: Size used for cache key. Tuple
519 @return: Cached data or None
520 @rtype: String
521 """
522
523 k = self._thumbKey(r, client_base, user_id, iid, size)
524 r = self._thumb_cache.get(k)
525 if r is None:
526 logger.debug(' fail: %s' % k)
527 else:
528 logger.debug('cached: %s' % k)
529 return r
530
531 - def clearThumb (self, r, client_base, user_id, iid, size=None):
532 """
533 Clears thumbnail from cache.
534
535 @param r: for cache key - Not used?
536 @param client_base: server_id for cache key
537 @param user_id: OMERO user ID to partition caching upon
538 @param iid: image ID for cache key
539 @param size: Size used for cache key. Tuple
540 @return: True
541 """
542 k = self._thumbKey(r, client_base, user_id, iid, size)
543 self._cache_clear(self._thumb_cache, k)
544 return True
545
546
547
548
549 - def _imageKey (self, r, client_base, img, z=0, t=0):
550 """
551 Returns a key for caching the Image, based on parameters above, including rendering settings
552 specified in the http request.
553
554 @param r: http request - get rendering params 'c', 'm', 'p'
555 @param client_base: server_id for cache key
556 @param img: L{omero.gateway.ImageWrapper} for ID
557 @param obj: Data to cache
558 @param size: Size used for cache key. Tuple
559 """
560
561 iid = img.getId()
562 if r:
563 r = r.REQUEST
564 c = FN_REGEX.sub('-',r.get('c', ''))
565 m = r.get('m', '')
566 p = r.get('p', '')
567 if p and not isinstance(omero.gateway.ImageWrapper.PROJECTIONS.get(p, -1),
568 omero.constants.projection.ProjectionType):
569 p = ''
570 q = r.get('q', '')
571 region = r.get('region', '')
572 tile = r.get('tile', '')
573 rv = 'img_%s/%s/%%s-c%s-m%s-q%s-r%s-t%s' % (client_base, str(iid), c, m, q, region, tile)
574 if p:
575 return rv % ('%s-%s' % (p, str(t)))
576 else:
577 return rv % ('%sx%s' % (str(z), str(t)))
578 else:
579 return 'img_%s/%s/' % (client_base, str(iid))
580
581 - def setImage (self, r, client_base, img, z, t, obj, ctx=''):
582 """
583 Puts image data into cache.
584
585 @param r: http request for cache key
586 @param client_base: server_id for cache key
587 @param img: ImageWrapper for cache key
588 @param z: Z index for cache key
589 @param t: T index for cache key
590 @param obj: Data to cache
591 @param ctx: Additional string for cache key
592 """
593
594 k = self._imageKey(r, client_base, img, z, t) + ctx
595 self._cache_set(self._img_cache, k, obj)
596 return True
597
598 - def getImage (self, r, client_base, img, z, t, ctx=''):
599 """
600 Gets image data from cache.
601
602 @param r: http request for cache key
603 @param client_base: server_id for cache key
604 @param img: ImageWrapper for cache key
605 @param z: Z index for cache key
606 @param t: T index for cache key
607 @param ctx: Additional string for cache key
608 @return: Image data
609 @rtype: String
610 """
611 k = self._imageKey(r, client_base, img, z, t) + ctx
612 r = self._img_cache.get(k)
613 if r is None:
614 logger.debug(' fail: %s' % k)
615 else:
616 logger.debug('cached: %s' % k)
617 return r
618
619 - def clearImage (self, r, client_base, user_id, img):
620 """
621 Clears image data from cache using default rendering settings (r=None) T and Z indexes ( = 0).
622 TODO: Doesn't clear any data stored WITH r, t, or z specified in cache key?
623 Also clears thumbnail (but not thumbs with size specified) and json data for this image.
624
625 @param r: http request for cache key
626 @param client_base: server_id for cache key
627 @param user_id: OMERO user ID to partition caching upon
628 @param img: ImageWrapper for cache key
629 @param obj: Data to cache
630 @param rtype: True
631 """
632
633 k = self._imageKey(None, client_base, img)
634 self._cache_clear(self._img_cache, k)
635
636 self.clearThumb(r, client_base, user_id, img.getId())
637
638 self.clearJson(client_base, img)
639 return True
640
642 """ Calls L{setImage} with '-sc' context """
643 return self.setImage(r, client_base, img, z, t, obj, '-sc')
644
646 """
647 Calls L{getImage} with '-sc' context
648 @rtype: String
649 """
650 return self.getImage(r, client_base, img, z, t, '-sc')
651
653 """ Calls L{setImage} with '-ometiff' context """
654 return self.setImage(r, client_base, img, 0, 0, obj, '-ometiff')
655
657 """
658 Calls L{getImage} with '-ometiff' context
659 @rtype: String
660 """
661 return self.getImage(r, client_base, img, 0, 0, '-ometiff')
662
663
664
665
666 - def _jsonKey (self, r, client_base, obj, ctx=''):
667 """
668 Creates a cache key for storing json data based on params above.
669
670 @param r: http request - not used
671 @param client_base: server_id
672 @param obj: ObjectWrapper
673 @param ctx: Additional string for cache key
674 @return: Cache key
675 @rtype: String
676 """
677
678 if obj:
679 return 'json_%s/%s_%s/%s' % (client_base, obj.OMERO_CLASS, obj.id, ctx)
680 else:
681 return 'json_%s/single/%s' % (client_base, ctx)
682
690
691 - def setDatasetContents (self, r, client_base, ds, data):
692 """
693 Adds data to the json cache using 'contents' as context
694
695 @param r: http request - not used
696 @param client_base: server_id for cache key
697 @param ds: ObjectWrapper for cache key
698 @param data: Data to cache
699 @rtype: True
700 """
701
702 k = self._jsonKey(r, client_base, ds, 'contents')
703 self._cache_set(self._json_cache, k, data)
704 return True
705
706 - def getDatasetContents (self, r, client_base, ds):
707 """
708 Gets data from the json cache using 'contents' as context
709
710 @param r: http request - not used
711 @param client_base: server_id for cache key
712 @param ds: ObjectWrapper for cache key
713 @rtype: String or None
714 """
715
716 k = self._jsonKey(r, client_base, ds, 'contents')
717 r = self._json_cache.get(k)
718 if r is None:
719 logger.debug(' fail: %s' % k)
720 else:
721 logger.debug('cached: %s' % k)
722 return r
723
724 - def clearDatasetContents (self, r, client_base, ds):
725 """
726 Clears data from the json cache using 'contents' as context
727
728 @param r: http request - not used
729 @param client_base: server_id for cache key
730 @param ds: ObjectWrapper for cache key
731 @rtype: True
732 """
733
734 k = self._jsonKey(r, client_base, ds, 'contents')
735 self._cache_clear(self._json_cache, k)
736 return True
737
738 webgateway_cache = WebGatewayCache(FileCache)
739
741 """ Class extends file to facilitate creation and deletion of lock file. """
742
744 """ creates a '.lock' file with the spicified file name and mode """
745 super(AutoLockFile, self).__init__(fn, mode)
746 self._lock = os.path.join(os.path.dirname(fn), '.lock')
747 file(self._lock, 'a').close()
748
750 """ tries to delete the lock file """
751 try:
752 os.remove(self._lock)
753 except:
754 pass
755
757 """ tries to delete the lock file and close the file """
758 try:
759 os.remove(self._lock)
760 except:
761 pass
762 super(AutoLockFile, self).close()
763
765 """
766 Class for handling creation of temporary files
767 """
768
770 """ Initialises class, setting the directory to be used for temp files. """
771 self._dir = tdir
772 if tdir and not os.path.exists(self._dir):
773 self._createdir()
774
776 """ Tries to create the directories required for the temp file base dir """
777 try:
778 os.makedirs(self._dir)
779 except OSError:
780 raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
781
783 """ Tries to delete all the temp files that have expired their cache timeout. """
784 now = time.time()
785 for f in os.listdir(self._dir):
786 try:
787 ts = os.path.join(self._dir, f, '.timestamp')
788 if os.path.exists(ts):
789 ft = float(file(ts).read()) + TMPDIR_TIME
790 else:
791 ft = float(f) + TMPDIR_TIME
792 if ft < now:
793 shutil.rmtree(os.path.join(self._dir, f), ignore_errors=True)
794 except ValueError:
795 continue
796
798 """
799 Creates a new directory using key as the dir name, and adds a timestamp file with it's
800 creation time. If key is not specified, use a unique key based on timestamp.
801
802 @param key: The new dir name
803 @return: Tuple of (path to new directory, key used)
804 """
805
806 if not self._dir:
807 return None, None
808 self._cleanup()
809 stamp = str(time.time())
810 if key is None:
811 dn = os.path.join(self._dir, stamp)
812 while os.path.exists(dn):
813 stamp = str(time.time())
814 dn = os.path.join(self._dir, stamp)
815 key = stamp
816 key = key.replace('/','_').decode('utf8').encode('ascii','ignore')
817 dn = os.path.join(self._dir, key)
818 if not os.path.isdir(dn):
819 os.makedirs(dn)
820 file(os.path.join(dn, '.timestamp'), 'w').write(stamp)
821 return dn, key
822
823 - def new (self, name, key=None):
824 """
825 Creates a new directory if needed, see L{newdir} and checks whether this contains a file 'name'. If not, a
826 file lock is created for this location and returned.
827
828 @param name: Name of file we want to create.
829 @param key: The new dir name
830 @return: Tuple of (abs path to new directory, relative path key/name, L{AutoFileLock} or True if exists)
831 """
832
833 if not self._dir:
834 return None, None, None
835 dn, stamp = self.newdir(key)
836 name = name.replace('/','_').decode('utf8').encode('ascii', 'ignore')
837 fn = os.path.join(dn, name)
838 rn = os.path.join(stamp, name)
839 lf = os.path.join(dn, '.lock')
840 cnt = 30
841 fsize = 0
842 while os.path.exists(lf) and cnt > 0:
843 time.sleep(1)
844 t = os.stat(fn)[stat.ST_SIZE]
845 if (t == fsize):
846 cnt -= 1
847 logger.debug('countdown %d' % cnt)
848 else:
849 fsize = t
850 cnt = 30
851 if cnt == 0:
852 return None, None, None
853 if os.path.exists(fn):
854 return fn, rn, True
855 return fn, rn, AutoLockFile(fn, 'wb')
856
857 webgateway_tempfile = WebGatewayTempFile()
858