Package omero :: Package util :: Module temp_files
[hide private]
[frames] | no frames]

Source Code for Module omero.util.temp_files

  1  #!/usr/bin/env python 
  2  # 
  3  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  4  # Use is subject to license terms supplied in LICENSE.txt 
  5  # 
  6  """ 
  7  OMERO Support for temporary files and directories 
  8  """ 
  9   
 10  import os 
 11  import sys 
 12  import atexit 
 13  import getpass 
 14  import logging 
 15  import tempfile 
 16  import threading 
 17  import traceback 
 18  import exceptions 
 19  import portalocker 
 20   
 21  from path import path 
 22  from omero.util import get_user_dir 
 23   
 24  # Activating logging at a static level 
 25  if "DEBUG" in os.environ: 
 26      from omero.util import configure_logging 
 27      configure_logging(loglevel=logging.DEBUG) 
 28   
 29  # TODO: 
 30  #  - locking for command-line cleanup 
 31  #  - plugin for cleaning unlocked files 
 32  #  - plugin for counting sizes, etc. 
 33  #  - decorator 
 34   
35 -class TempFileManager(object):
36 """ 37 Creates temporary files and folders and makes a best effort 38 to remove them on exit (or sooner). Typically only a single 39 instance of this class will exist ("manager" variable in this 40 module below) 41 """ 42
43 - def __init__(self, prefix = "omero"):
44 """ 45 Initializes a TempFileManager instance with a userDir containing 46 the given prefix value, or "omero" by default. Also registers 47 an atexit callback to call self.cleanup() on exit. 48 """ 49 self.logger = logging.getLogger("omero.util.TempFileManager") 50 self.is_win32 = ( sys.platform == "win32" ) 51 self.prefix = prefix 52 53 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username())) 54 """ 55 User-accessible directory of the form $TMPDIR/omero_$USERNAME. 56 If the given directory is not writable, an attempt is made 57 to use an alternative 58 """ 59 if not self.create(self.userdir) and not self.access(self.userdir): 60 i = 0 61 while i < 10: 62 t = path("%s_%s" % (self.userdir, i)) 63 if self.create(t) or self.access(t): 64 self.userdir = t 65 break 66 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir) 67 self.dir = self.userdir / self.pid() 68 """ 69 Directory under which all temporary files and folders will be created. 70 An attempt to remove a path not in this directory will lead to an 71 exception. 72 """ 73 74 # Now create the directory. If a later step throws an 75 # exception, we should try to rollback this change. 76 if not self.dir.exists(): 77 self.dir.makedirs() 78 self.logger.debug("Using temp dir: %s" % self.dir) 79 80 self.lock = None 81 try: 82 self.lock = open(str(self.dir / ".lock"), "a+") 83 """ 84 .lock file under self.dir which is used to prevent other 85 TempFileManager instances (also in other languages) from 86 cleaning up this directory. 87 """ 88 try: 89 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB) 90 atexit.register(self.cleanup) 91 except: 92 lock = self.lock 93 self.lock = None 94 if lock: 95 self.lock.close() 96 raise 97 finally: 98 try: 99 if not self.lock: 100 self.cleanup() 101 except: 102 self.logger.warn("Error on cleanup after error", exc_info = True)
103
104 - def cleanup(self):
105 """ 106 Releases self.lock and deletes self.dir. 107 The lock is released first since on some platforms like Windows 108 the lock file cannot be deleted even by the owner of the lock. 109 """ 110 try: 111 if self.lock: 112 self.lock.close() # Allow others access 113 except: 114 self.logger.error("Failed to release lock", exc_info = True) 115 self.clean_tempdir()
116
117 - def tmpdir(self):
118 """ 119 Returns a platform-specific user-writable temporary directory 120 121 First, the value of "OMERO_TEMPDIR" is attempted (if available), 122 then user's home directory, then the global temp director. 123 124 Typical errors for any of the possible temp locations are: 125 * non-existence 126 * inability to lock 127 128 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653 129 """ 130 locktest = None 131 132 omerotemp = os.environ.get("OMERO_TEMPDIR", None) 133 homeprop = None 134 try: 135 homeprop = get_user_dir() 136 except: 137 pass # ticket:3194 138 tempprop = tempfile.gettempdir() 139 targets = [omerotemp, homeprop, tempprop] 140 141 name = None 142 choice = None 143 locktest = None 144 145 for target in targets: 146 147 if target is None: 148 continue 149 150 if choice is not None: 151 break 152 153 try: 154 155 # 2805 156 omero_dir = path(target) / "omero" 157 if omero_dir.exists() and not omero_dir.isdir(): 158 self.logger.debug(""""omero" is not a directory: %s""" % omero_dir) 159 continue 160 tmp_dir = omero_dir / "tmp" 161 if tmp_dir.exists() and not tmp_dir.isdir(): 162 self.logger.debug(""""tmp" is not a directory: %s""" % tmp_dir) 163 continue 164 165 try: 166 167 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target) 168 locktest = open(name, "a+") 169 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB) 170 locktest.close() 171 locktest = None 172 choice = target 173 self.logger.debug("Chose gloabl tmpdir: %s", choice) 174 finally: 175 if locktest is not None: 176 try: 177 locktest.close() 178 except: 179 self.logger.warn("Failed to close locktest: %s", name, exc_info = True) 180 181 if name is not None: 182 try: 183 os.remove(name) 184 except: 185 self.logger.debug("Failed os.remove(%s)", name) 186 187 188 except exceptions.Exception, e: 189 if "Operation not permitted" in str(e) or \ 190 "Operation not supported" in str(e): 191 192 # This is the issue described in ticket:1653 193 # To prevent printing the warning, we just continue 194 # here. 195 self.logger.debug("%s does not support locking.", target) 196 else: 197 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True) 198 199 if choice is None: 200 raise exceptions.Exception("Could not find lockable tmp dir") 201 202 return path(choice) / "omero" / "tmp"
203
204 - def username(self):
205 """ 206 Returns the current OS-user's name 207 """ 208 try: 209 return getpass.getuser() # Uses environment variable or pwd 210 except ImportError: # No pwd on Windows 211 import win32api 212 return win32api.GetUserName()
213
214 - def pid(self):
215 """ 216 Returns some representation of the current process's id 217 """ 218 return str(os.getpid())
219
220 - def access(self, dir):
221 """ 222 Returns True if the current user can write to the given directory 223 """ 224 dir = str(dir) 225 return os.access(dir, os.W_OK)
226
227 - def create(self, dir):
228 """ 229 If the given directory doesn't exist, creates it (with mode 0700) and returns True. 230 Otherwise False. 231 """ 232 dir = path(dir) 233 if not dir.exists(): 234 dir.makedirs(0700) 235 return True 236 return False
237
238 - def gettempdir(self):
239 """ 240 Returns the directory under which all temporary 241 files and folders will be created. 242 """ 243 return self.dir
244
245 - def mkstemp(self, prefix, suffix, dir, text = False):
246 """ 247 Similar to tempfile.mkstemp name but immediately closes 248 the file descriptor returned and passes back just the name. 249 This prevents various Windows issues" 250 """ 251 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text) 252 self.logger.debug("Added file %s", name) 253 try: 254 os.close(fd) 255 except: 256 self.logger.warn("Failed to close fd %s" % fd) 257 return name
258
259 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
260 """ 261 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary 262 folders and files, respectively, under self.dir 263 """ 264 265 if folder: 266 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir) 267 self.logger.debug("Added folder %s", name) 268 else: 269 name = self.mkstemp(prefix, suffix, self.dir, text) 270 271 return path(name)
272
273 - def remove_path(self, name):
274 """ 275 If the given path is under self.dir, then it is deleted 276 whether file or folder. Otherwise, an exception is thrown. 277 """ 278 p = path(name) 279 parpath = p.parpath(self.dir) 280 if len(parpath) < 1: 281 raise exceptions.Exception("%s is not in %s" % (p, self.dir)) 282 283 if p.exists(): 284 if p.isdir(): 285 p.rmtree(onerror = self.on_rmtree) 286 self.logger.debug("Removed folder %s", name) 287 else: 288 p.remove() 289 self.logger.debug("Removed file %s", name)
290
291 - def clean_tempdir(self):
292 """ 293 Deletes self.dir 294 """ 295 dir = self.gettempdir() 296 if dir.exists(): 297 self.logger.debug("Removing tree: %s", dir) 298 dir.rmtree(onerror = self.on_rmtree)
299
300 - def clean_userdir(self):
301 """ 302 Attempts to delete all directories under self.userdir 303 other than the one owned by this process. If a directory 304 is locked, it is skipped. 305 """ 306 self.logger.debug("Cleaning user dir: %s" % self.userdir) 307 dirs = self.userdir.dirs() 308 for dir in dirs: 309 if str(dir) == str(self.dir): 310 self.logger.debug("Skipping self: %s", dir) 311 continue 312 lock = dir / ".lock" 313 if lock.exists(): #1962, on Windows this fails if lock is missing 314 f = open(str(lock),"r") 315 try: 316 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB) 317 f.close() # Must close for Windows, otherwise "...other process" 318 except: 319 print "Locked: %s" % dir 320 continue 321 dir.rmtree(onerror=self.on_rmtree) 322 print "Deleted: %s" % dir
323
324 - def on_rmtree(self, func, name, exc):
325 self.logger.error("rmtree error: %s('%s') => %s", func.__name__, name, exc[1])
326 327 manager = TempFileManager() 328 """ 329 Global TempFileManager instance for use by the current process and 330 registered with the atexit module for cleaning up all created files on exit. 331 Other instances can be created for specialized purposes. 332 """ 333
334 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
335 """ 336 Uses the global TempFileManager to create a temporary file. 337 """ 338 return manager.create_path(prefix, suffix, folder = folder)
339
340 -def remove_path(file):
341 """ 342 Removes the file from the global TempFileManager. The file will be deleted 343 if it still exists. 344 """ 345 return manager.remove_path(file)
346
347 -def gettempdir():
348 """ 349 Returns the dir value for the global TempFileManager. 350 """ 351 return manager.gettempdir()
352 353 if __name__ == "__main__": 354 355 from omero.util import configure_logging 356 357 if len(sys.argv) > 1: 358 args = sys.argv[1:] 359 else: 360 args = [] 361 362 # Debug may already be activated. See static block above. 363 if "--debug" in args and "DEBUG" not in os.environ: 364 configure_logging(loglevel=logging.DEBUG) 365 else: 366 configure_logging() 367 368 if "clean" in args: 369 manager.clean_userdir() 370 sys.exit(0) 371 elif "dir" in args: 372 print manager.gettempdir() 373 sys.exit(0) 374 elif "lock" in args: 375 print "Locking %s" % manager.gettempdir() 376 raw_input("Waiting on user input...") 377 sys.exit(0) 378 379 print "Usage: %s clean" % sys.argv[0] 380 print " or: %s dir " % sys.argv[0] 381 sys.exit(2) 382