1
2
3
4
5
6 """
7 OMERO Support for temporary files and directories
8 """
9
10 import os
11 import sys
12 import atexit
13 import getpass
14 import logging
15 import tempfile
16 import threading
17 import traceback
18 import exceptions
19 import portalocker
20
21 from path import path
22 from omero.util import get_user_dir
23
24
25 if "DEBUG" in os.environ:
26 from omero.util import configure_logging
27 configure_logging(loglevel=logging.DEBUG)
28
29
30
31
32
33
34
36 """
37 Creates temporary files and folders and makes a best effort
38 to remove them on exit (or sooner). Typically only a single
39 instance of this class will exist ("manager" variable in this
40 module below)
41 """
42
44 """
45 Initializes a TempFileManager instance with a userDir containing
46 the given prefix value, or "omero" by default. Also registers
47 an atexit callback to call self.cleanup() on exit.
48 """
49 self.logger = logging.getLogger("omero.util.TempFileManager")
50 self.is_win32 = ( sys.platform == "win32" )
51 self.prefix = prefix
52
53 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username()))
54 """
55 User-accessible directory of the form $TMPDIR/omero_$USERNAME.
56 If the given directory is not writable, an attempt is made
57 to use an alternative
58 """
59 if not self.create(self.userdir) and not self.access(self.userdir):
60 i = 0
61 while i < 10:
62 t = path("%s_%s" % (self.userdir, i))
63 if self.create(t) or self.access(t):
64 self.userdir = t
65 break
66 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir)
67 self.dir = self.userdir / self.pid()
68 """
69 Directory under which all temporary files and folders will be created.
70 An attempt to remove a path not in this directory will lead to an
71 exception.
72 """
73
74
75
76 if not self.dir.exists():
77 self.dir.makedirs()
78 self.logger.debug("Using temp dir: %s" % self.dir)
79
80 self.lock = None
81 try:
82 self.lock = open(str(self.dir / ".lock"), "a+")
83 """
84 .lock file under self.dir which is used to prevent other
85 TempFileManager instances (also in other languages) from
86 cleaning up this directory.
87 """
88 try:
89 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB)
90 atexit.register(self.cleanup)
91 except:
92 lock = self.lock
93 self.lock = None
94 if lock:
95 self.lock.close()
96 raise
97 finally:
98 try:
99 if not self.lock:
100 self.cleanup()
101 except:
102 self.logger.warn("Error on cleanup after error", exc_info = True)
103
105 """
106 Releases self.lock and deletes self.dir.
107 The lock is released first since on some platforms like Windows
108 the lock file cannot be deleted even by the owner of the lock.
109 """
110 try:
111 if self.lock:
112 self.lock.close()
113 except:
114 self.logger.error("Failed to release lock", exc_info = True)
115 self.clean_tempdir()
116
118 """
119 Returns a platform-specific user-writable temporary directory
120
121 First, the value of "OMERO_TEMPDIR" is attempted (if available),
122 then user's home directory, then the global temp director.
123
124 Typical errors for any of the possible temp locations are:
125 * non-existence
126 * inability to lock
127
128 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653
129 """
130 locktest = None
131
132 omerotemp = os.environ.get("OMERO_TEMPDIR", None)
133 homeprop = get_user_dir()
134 tempprop = tempfile.gettempdir()
135 targets = [omerotemp, homeprop, tempprop]
136
137 name = None
138 choice = None
139 locktest = None
140
141 for target in targets:
142
143 if target is None:
144 continue
145
146 if choice is not None:
147 break
148
149 try:
150 try:
151 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target)
152 locktest = open(name, "a+")
153 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB)
154 locktest.close()
155 locktest = None
156 choice = target
157 self.logger.debug("Chose gloabl tmpdir: %s", choice)
158 finally:
159 if locktest is not None:
160 try:
161 locktest.close()
162 except:
163 self.logger.warn("Failed to close locktest: %s", name, exc_info = True)
164
165 if name is not None:
166 try:
167 os.remove(name)
168 except:
169 self.logger.debug("Failed os.remove(%s)", name)
170
171
172 except exceptions.Exception, e:
173 if "Operation not permitted" in str(e) or \
174 "Operation not supported" in str(e):
175
176
177
178
179 self.logger.debug("%s does not support locking.", target)
180 else:
181 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True)
182
183 if choice is None:
184 raise exceptions.Exception("Could not find lockable tmp dir")
185
186 return path(choice) / "omero" / "tmp"
187
189 """
190 Returns the current OS-user's name
191 """
192 try:
193 return getpass.getuser()
194 except ImportError:
195 import win32api
196 return win32api.GetUserName()
197
199 """
200 Returns some representation of the current process's id
201 """
202 return str(os.getpid())
203
205 """
206 Returns True if the current user can write to the given directory
207 """
208 dir = str(dir)
209 return os.access(dir, os.W_OK)
210
212 """
213 If the given directory doesn't exist, creates it (with mode 0700) and returns True.
214 Otherwise False.
215 """
216 dir = path(dir)
217 if not dir.exists():
218 dir.makedirs(0700)
219 return True
220 return False
221
223 """
224 Returns the directory under which all temporary
225 files and folders will be created.
226 """
227 return self.dir
228
229 - def mkstemp(self, prefix, suffix, dir, text = False):
230 """
231 Similar to tempfile.mkstemp name but immediately closes
232 the file descriptor returned and passes back just the name.
233 This prevents various Windows issues"
234 """
235 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text)
236 self.logger.debug("Added file %s", name)
237 try:
238 os.close(fd)
239 except:
240 self.logger.warn("Failed to close fd %s" % fd)
241 return name
242
243 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
244 """
245 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary
246 folders and files, respectively, under self.dir
247 """
248
249 if folder:
250 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir)
251 self.logger.debug("Added folder %s", name)
252 else:
253 name = self.mkstemp(prefix, suffix, self.dir, text)
254
255 return path(name)
256
258 """
259 If the given path is under self.dir, then it is deleted
260 whether file or folder. Otherwise, an exception is thrown.
261 """
262 p = path(name)
263 parpath = p.parpath(self.dir)
264 if len(parpath) < 1:
265 raise exceptions.Exception("%s is not in %s" % (p, self.dir))
266
267 if p.exists():
268 if p.isdir():
269 p.rmtree(onerror = self.on_rmtree)
270 self.logger.debug("Removed folder %s", name)
271 else:
272 p.remove()
273 self.logger.debug("Removed file %s", name)
274
283
285 """
286 Attempts to delete all directories under self.userdir
287 other than the one owned by this process. If a directory
288 is locked, it is skipped.
289 """
290 self.logger.debug("Cleaning user dir: %s" % self.userdir)
291 dirs = self.userdir.dirs()
292 for dir in dirs:
293 if str(dir) == str(self.dir):
294 self.logger.debug("Skipping self: %s", dir)
295 continue
296 lock = dir / ".lock"
297 if lock.exists():
298 f = open(str(lock),"r")
299 try:
300 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB)
301 f.close()
302 except:
303 print "Locked: %s" % dir
304 continue
305 dir.rmtree(onerror=self.on_rmtree)
306 print "Deleted: %s" % dir
307
309 self.logger.error("rmtree error: %s('%s') => %s", func.__name__, name, exc[1])
310
311 manager = TempFileManager()
312 """
313 Global TempFileManager instance for use by the current process and
314 registered with the atexit module for cleaning up all created files on exit.
315 Other instances can be created for specialized purposes.
316 """
317
318 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
319 """
320 Uses the global TempFileManager to create a temporary file.
321 """
322 return manager.create_path(prefix, suffix, folder = folder)
323
325 """
326 Removes the file from the global TempFileManager. The file will be deleted
327 if it still exists.
328 """
329 return manager.remove_path(file)
330
332 """
333 Returns the dir value for the global TempFileManager.
334 """
335 return manager.gettempdir()
336
337 if __name__ == "__main__":
338
339 from omero.util import configure_logging
340
341 if len(sys.argv) > 1:
342 args = sys.argv[1:]
343 else:
344 args = []
345
346
347 if "--debug" in args and "DEBUG" not in os.environ:
348 configure_logging(loglevel=logging.DEBUG)
349 else:
350 configure_logging()
351
352 if "clean" in args:
353 manager.clean_userdir()
354 sys.exit(0)
355 elif "dir" in args:
356 print manager.gettempdir()
357 sys.exit(0)
358 elif "lock" in args:
359 print "Locking %s" % manager.gettempdir()
360 raw_input("Waiting on user input...")
361 sys.exit(0)
362
363 print "Usage: %s clean" % sys.argv[0]
364 print " or: %s dir " % sys.argv[0]
365 sys.exit(2)
366