Package omero :: Package util :: Module temp_files
[hide private]
[frames] | no frames]

Source Code for Module omero.util.temp_files

  1  #!/usr/bin/env python 
  2  # 
  3  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  4  # Use is subject to license terms supplied in LICENSE.txt 
  5  # 
  6  """ 
  7  OMERO Support for temporary files and directories 
  8  """ 
  9   
 10  import os 
 11  import sys 
 12  import atexit 
 13  import getpass 
 14  import logging 
 15  import tempfile 
 16  import threading 
 17  import traceback 
 18  import exceptions 
 19  import portalocker 
 20   
 21  from path import path 
 22   
 23  # Activating logging at a static level 
 24  if "DEBUG" in os.environ: 
 25      from omero.util import configure_logging 
 26      configure_logging(loglevel=logging.DEBUG) 
 27   
 28  # TODO: 
 29  #  - locking for command-line cleanup 
 30  #  - plugin for cleaning unlocked files 
 31  #  - plugin for counting sizes, etc. 
 32  #  - decorator 
 33   
34 -class TempFileManager(object):
35 """ 36 Creates temporary files and folders and makes a best effort 37 to remove them on exit (or sooner). Typically only a single 38 instance of this class will exist ("manager" variable in this 39 module below) 40 """ 41
42 - def __init__(self, prefix = "omero"):
43 """ 44 Initializes a TempFileManager instance with a userDir containing 45 the given prefix value, or "omero" by default. Also registers 46 an atexit callback to call self.cleanup() on exit. 47 """ 48 self.logger = logging.getLogger("omero.util.TempFileManager") 49 self.is_win32 = ( sys.platform == "win32" ) 50 self.prefix = prefix 51 52 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username())) 53 """ 54 User-accessible directory of the form $TMPDIR/omero_$USERNAME. 55 If the given directory is not writable, an attempt is made 56 to use an alternative 57 """ 58 if not self.create(self.userdir) and not self.access(self.userdir): 59 i = 0 60 while i < 10: 61 t = path("%s_%s" % (self.userdir, i)) 62 if self.create(t) or self.access(t): 63 self.userdir = t 64 break 65 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir) 66 self.dir = self.userdir / self.pid() 67 """ 68 Directory under which all temporary files and folders will be created. 69 An attempt to remove a path not in this directory will lead to an 70 exception. 71 """ 72 73 # Now create the directory. If a later step throws an 74 # exception, we should try to rollback this change. 75 if not self.dir.exists(): 76 self.dir.makedirs() 77 self.logger.debug("Using temp dir: %s" % self.dir) 78 79 self.lock = None 80 try: 81 self.lock = open(str(self.dir / ".lock"), "a+") 82 """ 83 .lock file under self.dir which is used to prevent other 84 TempFileManager instances (also in other languages) from 85 cleaning up this directory. 86 """ 87 try: 88 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB) 89 atexit.register(self.cleanup) 90 except: 91 lock = self.lock 92 self.lock = None 93 if lock: 94 self.lock.close() 95 raise 96 finally: 97 try: 98 if not self.lock: 99 self.cleanup() 100 except: 101 self.logger.warn("Error on cleanup after error", exc_info = True)
102
103 - def cleanup(self):
104 """ 105 Releases self.lock and deletes self.dir. 106 The lock is released first since on some platforms like Windows 107 the lock file cannot be deleted even by the owner of the lock. 108 """ 109 try: 110 if self.lock: 111 self.lock.close() # Allow others access 112 except: 113 self.logger.error("Failed to release lock", exc_info = True) 114 self.clean_tempdir()
115
116 - def tmpdir(self):
117 """ 118 Returns a platform-specific user-writable temporary directory 119 120 First, the value of "OMERO_TEMPDIR" is attempted (if available), 121 then user's home directory, then the global temp director. 122 123 Typical errors for any of the possible temp locations are: 124 * non-existence 125 * inability to lock 126 127 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653 128 """ 129 locktest = None 130 131 omerotemp = os.environ.get("OMERO_TEMPDIR", None) 132 try: 133 from win32com.shell import shellcon, shell 134 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 135 except ImportError: 136 homeprop = os.path.expanduser("~") 137 tempprop = tempfile.gettempdir() 138 targets = [omerotemp, homeprop, tempprop] 139 140 for target in targets: 141 142 if target is None: 143 continue 144 145 try: 146 try: 147 testdir = path(target) 148 fd, name = tempfile.mkstemp(prefix=".lock_test", suffix=".tmp", dir=str(testdir)) 149 locktest = open(name, "a+") 150 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB) 151 locktest.close() 152 self.logger.debug("Chose gloabl tmpdir: %s", testdir) 153 break 154 finally: 155 if locktest is not None: 156 try: 157 locktest.close() 158 os.close(fd) 159 except: 160 self.logger.warn("Failed to remove lock test: %s", name, exc_info = True) 161 162 except exceptions.Exception, e: 163 if "Operation not permitted" in str(e) or \ 164 "Operation not supported" in str(e): 165 166 # This is the issue described in ticket:1653 167 # To prevent printing the warning, we just continue 168 # here. 169 self.logger.debug("%s does not support locking.", target) 170 continue 171 else: 172 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True) 173 174 if locktest is None: 175 raise exceptions.Exception("Could not find lockable tmp dir") 176 177 return testdir / "omero" / "tmp"
178
179 - def username(self):
180 """ 181 Returns the current OS-user's name 182 """ 183 try: 184 return getpass.getuser() # Uses environment variable or pwd 185 except ImportError: # No pwd on Windows 186 import win32api 187 return win32api.GetUserName()
188
189 - def pid(self):
190 """ 191 Returns some representation of the current process's id 192 """ 193 return str(os.getpid())
194
195 - def access(self, dir):
196 """ 197 Returns True if the current user can write to the given directory 198 """ 199 dir = str(dir) 200 return os.access(dir, os.W_OK)
201
202 - def create(self, dir):
203 """ 204 If the given directory doesn't exist, creates it (with mode 0700) and returns True. 205 Otherwise False. 206 """ 207 dir = path(dir) 208 if not dir.exists(): 209 dir.makedirs(0700) 210 return True 211 return False
212
213 - def gettempdir(self):
214 """ 215 Returns the directory under which all temporary 216 files and folders will be created. 217 """ 218 return self.dir
219
220 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
221 """ 222 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary 223 folders and files, respectively, under self.dir 224 """ 225 226 if folder: 227 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir) 228 self.logger.debug("Added folder %s", name) 229 else: 230 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = self.dir, text = text) 231 self.logger.debug("Added file %s", name) 232 try: 233 os.close(fd) 234 except: 235 self.logger.warn("Failed to close fd %s" % fd) 236 237 return path(name)
238
239 - def remove_path(self, name):
240 """ 241 If the given path is under self.dir, then it is deleted 242 whether file or folder. Otherwise, an exception is thrown. 243 """ 244 p = path(name) 245 parpath = p.parpath(self.dir) 246 if len(parpath) < 1: 247 raise exceptions.Exception("%s is not in %s" % (p, self.dir)) 248 249 if p.exists(): 250 if p.isdir(): 251 p.rmtree(onerror = self.on_rmtree) 252 self.logger.debug("Removed folder %s", name) 253 else: 254 p.remove() 255 self.logger.debug("Removed file %s", name)
256
257 - def clean_tempdir(self):
258 """ 259 Deletes self.dir 260 """ 261 dir = self.gettempdir() 262 if dir.exists(): 263 self.logger.debug("Removing tree: %s", dir) 264 dir.rmtree(onerror = self.on_rmtree)
265
266 - def clean_userdir(self):
267 """ 268 Attempts to delete all directories under self.userdir 269 other than the one owned by this process. If a directory 270 is locked, it is skipped. 271 """ 272 self.logger.debug("Cleaning user dir: %s" % self.userdir) 273 dirs = self.userdir.dirs() 274 for dir in dirs: 275 if str(dir) == str(self.dir): 276 self.logger.debug("Skipping self: %s", dir) 277 continue 278 lock = dir / ".lock" 279 f = open(str(lock),"r") 280 try: 281 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB) 282 except: 283 print "Locked: %s" % dir 284 continue 285 dir.rmtree(self.on_rmtree) 286 print "Deleted: %s" % dir
287
288 - def on_rmtree(self, func, name, exc):
289 self.logger.error("rmtree error: %s('%s') => %s", func.__name__, name, exc[1])
290 291 manager = TempFileManager() 292 """ 293 Global TempFileManager instance for use by the current process and 294 registered with the atexit module for cleaning up all created files on exit. 295 Other instances can be created for specialized purposes. 296 """ 297
298 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
299 """ 300 Uses the global TempFileManager to create a temporary file. 301 """ 302 return manager.create_path(prefix, suffix, folder = folder)
303
304 -def remove_path(file):
305 """ 306 Removes the file from the global TempFileManager. The file will be deleted 307 if it still exists. 308 """ 309 return manager.remove_path(file)
310
311 -def gettempdir():
312 """ 313 Returns the dir value for the global TempFileManager. 314 """ 315 return manager.gettempdir()
316 317 if __name__ == "__main__": 318 319 from omero.util import configure_logging 320 321 if len(sys.argv) > 1: 322 args = sys.argv[1:] 323 else: 324 args = [] 325 326 # Debug may already be activated. See static block above. 327 if "--debug" in args and "DEBUG" not in os.environ: 328 configure_logging(loglevel=logging.DEBUG) 329 else: 330 configure_logging() 331 332 if "clean" in args: 333 manager.clean_userdir() 334 sys.exit(0) 335 elif "dir" in args: 336 print manager.gettempdir() 337 sys.exit(0) 338 elif "lock" in args: 339 print "Locking %s" % manager.gettempdir() 340 raw_input("Waiting on user input...") 341 sys.exit(0) 342 343 print "Usage: %s clean" % sys.argv[0] 344 print " or: %s dir " % sys.argv[0] 345 sys.exit(2) 346