Package omero :: Package util :: Module temp_files
[hide private]
[frames] | no frames]

Source Code for Module omero.util.temp_files

  1  #!/usr/bin/env python 
  2  # 
  3  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  4  # Use is subject to license terms supplied in LICENSE.txt 
  5  # 
  6  """ 
  7  OMERO Support for temporary files and directories 
  8  """ 
  9   
 10  import os 
 11  import sys 
 12  import atexit 
 13  import getpass 
 14  import logging 
 15  import tempfile 
 16  import threading 
 17  import traceback 
 18  import exceptions 
 19  import portalocker 
 20   
 21  from path import path 
 22  from omero.util import get_user_dir 
 23   
 24  # Activating logging at a static level 
 25  if "DEBUG" in os.environ: 
 26      from omero.util import configure_logging 
 27      configure_logging(loglevel=logging.DEBUG) 
 28   
 29  # TODO: 
 30  #  - locking for command-line cleanup 
 31  #  - plugin for cleaning unlocked files 
 32  #  - plugin for counting sizes, etc. 
 33  #  - decorator 
 34   
35 -class TempFileManager(object):
36 """ 37 Creates temporary files and folders and makes a best effort 38 to remove them on exit (or sooner). Typically only a single 39 instance of this class will exist ("manager" variable in this 40 module below) 41 """ 42
43 - def __init__(self, prefix = "omero"):
44 """ 45 Initializes a TempFileManager instance with a userDir containing 46 the given prefix value, or "omero" by default. Also registers 47 an atexit callback to call self.cleanup() on exit. 48 """ 49 self.logger = logging.getLogger("omero.util.TempFileManager") 50 self.is_win32 = ( sys.platform == "win32" ) 51 self.prefix = prefix 52 53 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username())) 54 """ 55 User-accessible directory of the form $TMPDIR/omero_$USERNAME. 56 If the given directory is not writable, an attempt is made 57 to use an alternative 58 """ 59 if not self.create(self.userdir) and not self.access(self.userdir): 60 i = 0 61 while i < 10: 62 t = path("%s_%s" % (self.userdir, i)) 63 if self.create(t) or self.access(t): 64 self.userdir = t 65 break 66 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir) 67 self.dir = self.userdir / self.pid() 68 """ 69 Directory under which all temporary files and folders will be created. 70 An attempt to remove a path not in this directory will lead to an 71 exception. 72 """ 73 74 # Now create the directory. If a later step throws an 75 # exception, we should try to rollback this change. 76 if not self.dir.exists(): 77 self.dir.makedirs() 78 self.logger.debug("Using temp dir: %s" % self.dir) 79 80 self.lock = None 81 try: 82 self.lock = open(str(self.dir / ".lock"), "a+") 83 """ 84 .lock file under self.dir which is used to prevent other 85 TempFileManager instances (also in other languages) from 86 cleaning up this directory. 87 """ 88 try: 89 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB) 90 atexit.register(self.cleanup) 91 except: 92 lock = self.lock 93 self.lock = None 94 if lock: 95 self.lock.close() 96 raise 97 finally: 98 try: 99 if not self.lock: 100 self.cleanup() 101 except: 102 self.logger.warn("Error on cleanup after error", exc_info = True)
103
104 - def cleanup(self):
105 """ 106 Releases self.lock and deletes self.dir. 107 The lock is released first since on some platforms like Windows 108 the lock file cannot be deleted even by the owner of the lock. 109 """ 110 try: 111 if self.lock: 112 self.lock.close() # Allow others access 113 except: 114 self.logger.error("Failed to release lock", exc_info = True) 115 self.clean_tempdir()
116
117 - def tmpdir(self):
118 """ 119 Returns a platform-specific user-writable temporary directory 120 121 First, the value of "OMERO_TEMPDIR" is attempted (if available), 122 then user's home directory, then the global temp director. 123 124 Typical errors for any of the possible temp locations are: 125 * non-existence 126 * inability to lock 127 128 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653 129 """ 130 locktest = None 131 132 omerotemp = os.environ.get("OMERO_TEMPDIR", None) 133 homeprop = get_user_dir() 134 tempprop = tempfile.gettempdir() 135 targets = [omerotemp, homeprop, tempprop] 136 137 name = None 138 choice = None 139 locktest = None 140 141 for target in targets: 142 143 if target is None: 144 continue 145 146 if choice is not None: 147 break 148 149 try: 150 try: 151 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target) 152 locktest = open(name, "a+") 153 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB) 154 locktest.close() 155 locktest = None 156 choice = target 157 self.logger.debug("Chose gloabl tmpdir: %s", choice) 158 finally: 159 if locktest is not None: 160 try: 161 locktest.close() 162 except: 163 self.logger.warn("Failed to close locktest: %s", name, exc_info = True) 164 165 if name is not None: 166 try: 167 os.remove(name) 168 except: 169 self.logger.debug("Failed os.remove(%s)", name) 170 171 172 except exceptions.Exception, e: 173 if "Operation not permitted" in str(e) or \ 174 "Operation not supported" in str(e): 175 176 # This is the issue described in ticket:1653 177 # To prevent printing the warning, we just continue 178 # here. 179 self.logger.debug("%s does not support locking.", target) 180 else: 181 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True) 182 183 if choice is None: 184 raise exceptions.Exception("Could not find lockable tmp dir") 185 186 return path(choice) / "omero" / "tmp"
187
188 - def username(self):
189 """ 190 Returns the current OS-user's name 191 """ 192 try: 193 return getpass.getuser() # Uses environment variable or pwd 194 except ImportError: # No pwd on Windows 195 import win32api 196 return win32api.GetUserName()
197
198 - def pid(self):
199 """ 200 Returns some representation of the current process's id 201 """ 202 return str(os.getpid())
203
204 - def access(self, dir):
205 """ 206 Returns True if the current user can write to the given directory 207 """ 208 dir = str(dir) 209 return os.access(dir, os.W_OK)
210
211 - def create(self, dir):
212 """ 213 If the given directory doesn't exist, creates it (with mode 0700) and returns True. 214 Otherwise False. 215 """ 216 dir = path(dir) 217 if not dir.exists(): 218 dir.makedirs(0700) 219 return True 220 return False
221
222 - def gettempdir(self):
223 """ 224 Returns the directory under which all temporary 225 files and folders will be created. 226 """ 227 return self.dir
228
229 - def mkstemp(self, prefix, suffix, dir, text = False):
230 """ 231 Similar to tempfile.mkstemp name but immediately closes 232 the file descriptor returned and passes back just the name. 233 This prevents various Windows issues" 234 """ 235 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text) 236 self.logger.debug("Added file %s", name) 237 try: 238 os.close(fd) 239 except: 240 self.logger.warn("Failed to close fd %s" % fd) 241 return name
242
243 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
244 """ 245 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary 246 folders and files, respectively, under self.dir 247 """ 248 249 if folder: 250 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir) 251 self.logger.debug("Added folder %s", name) 252 else: 253 name = self.mkstemp(prefix, suffix, self.dir, text) 254 255 return path(name)
256
257 - def remove_path(self, name):
258 """ 259 If the given path is under self.dir, then it is deleted 260 whether file or folder. Otherwise, an exception is thrown. 261 """ 262 p = path(name) 263 parpath = p.parpath(self.dir) 264 if len(parpath) < 1: 265 raise exceptions.Exception("%s is not in %s" % (p, self.dir)) 266 267 if p.exists(): 268 if p.isdir(): 269 p.rmtree(onerror = self.on_rmtree) 270 self.logger.debug("Removed folder %s", name) 271 else: 272 p.remove() 273 self.logger.debug("Removed file %s", name)
274
275 - def clean_tempdir(self):
276 """ 277 Deletes self.dir 278 """ 279 dir = self.gettempdir() 280 if dir.exists(): 281 self.logger.debug("Removing tree: %s", dir) 282 dir.rmtree(onerror = self.on_rmtree)
283
284 - def clean_userdir(self):
285 """ 286 Attempts to delete all directories under self.userdir 287 other than the one owned by this process. If a directory 288 is locked, it is skipped. 289 """ 290 self.logger.debug("Cleaning user dir: %s" % self.userdir) 291 dirs = self.userdir.dirs() 292 for dir in dirs: 293 if str(dir) == str(self.dir): 294 self.logger.debug("Skipping self: %s", dir) 295 continue 296 lock = dir / ".lock" 297 if lock.exists(): #1962, on Windows this fails if lock is missing 298 f = open(str(lock),"r") 299 try: 300 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB) 301 f.close() # Must close for Windows, otherwise "...other process" 302 except: 303 print "Locked: %s" % dir 304 continue 305 dir.rmtree(onerror=self.on_rmtree) 306 print "Deleted: %s" % dir
307
308 - def on_rmtree(self, func, name, exc):
309 self.logger.error("rmtree error: %s('%s') => %s", func.__name__, name, exc[1])
310 311 manager = TempFileManager() 312 """ 313 Global TempFileManager instance for use by the current process and 314 registered with the atexit module for cleaning up all created files on exit. 315 Other instances can be created for specialized purposes. 316 """ 317
318 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
319 """ 320 Uses the global TempFileManager to create a temporary file. 321 """ 322 return manager.create_path(prefix, suffix, folder = folder)
323
324 -def remove_path(file):
325 """ 326 Removes the file from the global TempFileManager. The file will be deleted 327 if it still exists. 328 """ 329 return manager.remove_path(file)
330
331 -def gettempdir():
332 """ 333 Returns the dir value for the global TempFileManager. 334 """ 335 return manager.gettempdir()
336 337 if __name__ == "__main__": 338 339 from omero.util import configure_logging 340 341 if len(sys.argv) > 1: 342 args = sys.argv[1:] 343 else: 344 args = [] 345 346 # Debug may already be activated. See static block above. 347 if "--debug" in args and "DEBUG" not in os.environ: 348 configure_logging(loglevel=logging.DEBUG) 349 else: 350 configure_logging() 351 352 if "clean" in args: 353 manager.clean_userdir() 354 sys.exit(0) 355 elif "dir" in args: 356 print manager.gettempdir() 357 sys.exit(0) 358 elif "lock" in args: 359 print "Locking %s" % manager.gettempdir() 360 raw_input("Waiting on user input...") 361 sys.exit(0) 362 363 print "Usage: %s clean" % sys.argv[0] 364 print " or: %s dir " % sys.argv[0] 365 sys.exit(2) 366