Package omero :: Package util :: Module temp_files
[hide private]
[frames] | no frames]

Source Code for Module omero.util.temp_files

  1  #!/usr/bin/env python 
  2  # 
  3  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  4  # Use is subject to license terms supplied in LICENSE.txt 
  5  # 
  6  """ 
  7  OMERO Support for temporary files and directories 
  8  """ 
  9   
 10  import os 
 11  import sys 
 12  import atexit 
 13  import logging 
 14  import tempfile 
 15  import threading 
 16  import traceback 
 17  import exceptions 
 18  import portalocker 
 19   
 20  from path import path 
 21  from omero.util import get_user_dir, get_user 
 22   
 23  # Activating logging at a static level 
 24  if "DEBUG" in os.environ: 
 25      from omero.util import configure_logging 
 26      configure_logging(loglevel=logging.DEBUG) 
 27   
 28  # TODO: 
 29  #  - locking for command-line cleanup 
 30  #  - plugin for cleaning unlocked files 
 31  #  - plugin for counting sizes, etc. 
 32  #  - decorator 
 33   
34 -class TempFileManager(object):
35 """ 36 Creates temporary files and folders and makes a best effort 37 to remove them on exit (or sooner). Typically only a single 38 instance of this class will exist ("manager" variable in this 39 module below) 40 """ 41
42 - def __init__(self, prefix = "omero"):
43 """ 44 Initializes a TempFileManager instance with a userDir containing 45 the given prefix value, or "omero" by default. Also registers 46 an atexit callback to call self.cleanup() on exit. 47 """ 48 self.logger = logging.getLogger("omero.util.TempFileManager") 49 self.is_win32 = ( sys.platform == "win32" ) 50 self.prefix = prefix 51 52 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username())) 53 """ 54 User-accessible directory of the form $TMPDIR/omero_$USERNAME. 55 If the given directory is not writable, an attempt is made 56 to use an alternative 57 """ 58 if not self.create(self.userdir) and not self.access(self.userdir): 59 i = 0 60 while i < 10: 61 t = path("%s_%s" % (self.userdir, i)) 62 if self.create(t) or self.access(t): 63 self.userdir = t 64 break 65 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir) 66 self.dir = self.userdir / self.pid() 67 """ 68 Directory under which all temporary files and folders will be created. 69 An attempt to remove a path not in this directory will lead to an 70 exception. 71 """ 72 73 # Now create the directory. If a later step throws an 74 # exception, we should try to rollback this change. 75 if not self.dir.exists(): 76 self.dir.makedirs() 77 self.logger.debug("Using temp dir: %s" % self.dir) 78 79 self.lock = None 80 try: 81 self.lock = open(str(self.dir / ".lock"), "a+") 82 """ 83 .lock file under self.dir which is used to prevent other 84 TempFileManager instances (also in other languages) from 85 cleaning up this directory. 86 """ 87 try: 88 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB) 89 atexit.register(self.cleanup) 90 except: 91 lock = self.lock 92 self.lock = None 93 if lock: 94 self.lock.close() 95 raise 96 finally: 97 try: 98 if not self.lock: 99 self.cleanup() 100 except: 101 self.logger.warn("Error on cleanup after error", exc_info = True)
102
103 - def cleanup(self):
104 """ 105 Releases self.lock and deletes self.dir. 106 The lock is released first since on some platforms like Windows 107 the lock file cannot be deleted even by the owner of the lock. 108 """ 109 try: 110 if self.lock: 111 self.lock.close() # Allow others access 112 except: 113 self.logger.error("Failed to release lock", exc_info = True) 114 self.clean_tempdir()
115
116 - def tmpdir(self):
117 """ 118 Returns a platform-specific user-writable temporary directory 119 120 First, the value of "OMERO_TEMPDIR" is attempted (if available), 121 then user's home directory, then the global temp director. 122 123 Typical errors for any of the possible temp locations are: 124 * non-existence 125 * inability to lock 126 127 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653 128 """ 129 locktest = None 130 131 omerotemp = os.environ.get("OMERO_TEMPDIR", None) 132 homeprop = None 133 try: 134 homeprop = get_user_dir() 135 except: 136 pass # ticket:3194, ticket:5583 137 tempprop = tempfile.gettempdir() 138 targets = [omerotemp, homeprop, tempprop] 139 140 name = None 141 choice = None 142 locktest = None 143 144 for target in targets: 145 146 if target is None: 147 continue 148 149 if choice is not None: 150 break 151 152 try: 153 154 # 2805 155 omero_dir = path(target) / "omero" 156 if omero_dir.exists() and not omero_dir.isdir(): 157 self.logger.debug(""""omero" is not a directory: %s""" % omero_dir) 158 continue 159 tmp_dir = omero_dir / "tmp" 160 if tmp_dir.exists() and not tmp_dir.isdir(): 161 self.logger.debug(""""tmp" is not a directory: %s""" % tmp_dir) 162 continue 163 164 try: 165 166 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target) 167 locktest = open(name, "a+") 168 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB) 169 locktest.close() 170 locktest = None 171 choice = target 172 self.logger.debug("Chose gloabl tmpdir: %s", choice) 173 finally: 174 if locktest is not None: 175 try: 176 locktest.close() 177 except: 178 self.logger.warn("Failed to close locktest: %s", name, exc_info = True) 179 180 if name is not None: 181 try: 182 os.remove(name) 183 except: 184 self.logger.debug("Failed os.remove(%s)", name) 185 186 187 except exceptions.Exception, e: 188 if "Operation not permitted" in str(e) or \ 189 "Operation not supported" in str(e): 190 191 # This is the issue described in ticket:1653 192 # To prevent printing the warning, we just continue 193 # here. 194 self.logger.debug("%s does not support locking.", target) 195 else: 196 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True) 197 198 if choice is None: 199 raise exceptions.Exception("Could not find lockable tmp dir") 200 201 return path(choice) / "omero" / "tmp"
202
203 - def username(self):
204 """ 205 Returns the current OS-user's name 206 """ 207 return get_user("Unknown")
208
209 - def pid(self):
210 """ 211 Returns some representation of the current process's id 212 """ 213 return str(os.getpid())
214
215 - def access(self, dir):
216 """ 217 Returns True if the current user can write to the given directory 218 """ 219 dir = str(dir) 220 return os.access(dir, os.W_OK)
221
222 - def create(self, dir):
223 """ 224 If the given directory doesn't exist, creates it (with mode 0700) and returns True. 225 Otherwise False. 226 """ 227 dir = path(dir) 228 if not dir.exists(): 229 dir.makedirs(0700) 230 return True 231 return False
232
233 - def gettempdir(self):
234 """ 235 Returns the directory under which all temporary 236 files and folders will be created. 237 """ 238 return self.dir
239
240 - def mkstemp(self, prefix, suffix, dir, text = False):
241 """ 242 Similar to tempfile.mkstemp name but immediately closes 243 the file descriptor returned and passes back just the name. 244 This prevents various Windows issues" 245 """ 246 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text) 247 self.logger.debug("Added file %s", name) 248 try: 249 os.close(fd) 250 except: 251 self.logger.warn("Failed to close fd %s" % fd) 252 return name
253
254 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
255 """ 256 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary 257 folders and files, respectively, under self.dir 258 """ 259 260 if folder: 261 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir) 262 self.logger.debug("Added folder %s", name) 263 else: 264 name = self.mkstemp(prefix, suffix, self.dir, text) 265 266 return path(name)
267
268 - def remove_path(self, name):
269 """ 270 If the given path is under self.dir, then it is deleted 271 whether file or folder. Otherwise, an exception is thrown. 272 """ 273 p = path(name) 274 parpath = p.parpath(self.dir) 275 if len(parpath) < 1: 276 raise exceptions.Exception("%s is not in %s" % (p, self.dir)) 277 278 if p.exists(): 279 if p.isdir(): 280 p.rmtree(onerror = self.on_rmtree) 281 self.logger.debug("Removed folder %s", name) 282 else: 283 p.remove() 284 self.logger.debug("Removed file %s", name)
285
286 - def clean_tempdir(self):
287 """ 288 Deletes self.dir 289 """ 290 dir = self.gettempdir() 291 if dir.exists(): 292 self.logger.debug("Removing tree: %s", dir) 293 dir.rmtree(onerror = self.on_rmtree)
294
295 - def clean_userdir(self):
296 """ 297 Attempts to delete all directories under self.userdir 298 other than the one owned by this process. If a directory 299 is locked, it is skipped. 300 """ 301 self.logger.debug("Cleaning user dir: %s" % self.userdir) 302 dirs = self.userdir.dirs() 303 for dir in dirs: 304 if str(dir) == str(self.dir): 305 self.logger.debug("Skipping self: %s", dir) 306 continue 307 lock = dir / ".lock" 308 if lock.exists(): #1962, on Windows this fails if lock is missing 309 f = open(str(lock),"r") 310 try: 311 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB) 312 f.close() # Must close for Windows, otherwise "...other process" 313 except: 314 print "Locked: %s" % dir 315 continue 316 dir.rmtree(onerror=self.on_rmtree) 317 print "Deleted: %s" % dir
318
319 - def on_rmtree(self, func, name, exc):
320 self.logger.error("rmtree error: %s('%s') => %s", func.__name__, name, exc[1])
321 322 manager = TempFileManager() 323 """ 324 Global TempFileManager instance for use by the current process and 325 registered with the atexit module for cleaning up all created files on exit. 326 Other instances can be created for specialized purposes. 327 """ 328
329 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
330 """ 331 Uses the global TempFileManager to create a temporary file. 332 """ 333 return manager.create_path(prefix, suffix, folder = folder)
334
335 -def remove_path(file):
336 """ 337 Removes the file from the global TempFileManager. The file will be deleted 338 if it still exists. 339 """ 340 return manager.remove_path(file)
341
342 -def gettempdir():
343 """ 344 Returns the dir value for the global TempFileManager. 345 """ 346 return manager.gettempdir()
347 348 if __name__ == "__main__": 349 350 from omero.util import configure_logging 351 352 if len(sys.argv) > 1: 353 args = sys.argv[1:] 354 else: 355 args = [] 356 357 # Debug may already be activated. See static block above. 358 if "--debug" in args and "DEBUG" not in os.environ: 359 configure_logging(loglevel=logging.DEBUG) 360 else: 361 configure_logging() 362 363 if "clean" in args: 364 manager.clean_userdir() 365 sys.exit(0) 366 elif "dir" in args: 367 print manager.gettempdir() 368 sys.exit(0) 369 elif "lock" in args: 370 print "Locking %s" % manager.gettempdir() 371 raw_input("Waiting on user input...") 372 sys.exit(0) 373 374 print "Usage: %s clean" % sys.argv[0] 375 print " or: %s dir " % sys.argv[0] 376 sys.exit(2) 377