1
2
3
4
5
6 """
7 OMERO Support for temporary files and directories
8 """
9
10 import os
11 import sys
12 import atexit
13 import logging
14 import tempfile
15 import threading
16 import traceback
17 import exceptions
18 import portalocker
19
20 from path import path
21 from omero.util import get_user_dir, get_user
22
23
24 if "DEBUG" in os.environ:
25 from omero.util import configure_logging
26 configure_logging(loglevel=logging.DEBUG)
27
28
29
30
31
32
33
35 """
36 Creates temporary files and folders and makes a best effort
37 to remove them on exit (or sooner). Typically only a single
38 instance of this class will exist ("manager" variable in this
39 module below)
40 """
41
43 """
44 Initializes a TempFileManager instance with a userDir containing
45 the given prefix value, or "omero" by default. Also registers
46 an atexit callback to call self.cleanup() on exit.
47 """
48 self.logger = logging.getLogger("omero.util.TempFileManager")
49 self.is_win32 = ( sys.platform == "win32" )
50 self.prefix = prefix
51
52 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username()))
53 """
54 User-accessible directory of the form $TMPDIR/omero_$USERNAME.
55 If the given directory is not writable, an attempt is made
56 to use an alternative
57 """
58 if not self.create(self.userdir) and not self.access(self.userdir):
59 i = 0
60 while i < 10:
61 t = path("%s_%s" % (self.userdir, i))
62 if self.create(t) or self.access(t):
63 self.userdir = t
64 break
65 raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir)
66 self.dir = self.userdir / self.pid()
67 """
68 Directory under which all temporary files and folders will be created.
69 An attempt to remove a path not in this directory will lead to an
70 exception.
71 """
72
73
74
75 if not self.dir.exists():
76 self.dir.makedirs()
77 self.logger.debug("Using temp dir: %s" % self.dir)
78
79 self.lock = None
80 try:
81 self.lock = open(str(self.dir / ".lock"), "a+")
82 """
83 .lock file under self.dir which is used to prevent other
84 TempFileManager instances (also in other languages) from
85 cleaning up this directory.
86 """
87 try:
88 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB)
89 atexit.register(self.cleanup)
90 except:
91 lock = self.lock
92 self.lock = None
93 if lock:
94 self.lock.close()
95 raise
96 finally:
97 try:
98 if not self.lock:
99 self.cleanup()
100 except:
101 self.logger.warn("Error on cleanup after error", exc_info = True)
102
104 """
105 Releases self.lock and deletes self.dir.
106 The lock is released first since on some platforms like Windows
107 the lock file cannot be deleted even by the owner of the lock.
108 """
109 try:
110 if self.lock:
111 self.lock.close()
112 except:
113 self.logger.error("Failed to release lock", exc_info = True)
114 self.clean_tempdir()
115
117 """
118 Returns a platform-specific user-writable temporary directory
119
120 First, the value of "OMERO_TEMPDIR" is attempted (if available),
121 then user's home directory, then the global temp director.
122
123 Typical errors for any of the possible temp locations are:
124 * non-existence
125 * inability to lock
126
127 See: https://trac.openmicroscopy.org.uk/omero/ticket/1653
128 """
129 locktest = None
130
131 omerotemp = os.environ.get("OMERO_TEMPDIR", None)
132 homeprop = None
133 try:
134 homeprop = get_user_dir()
135 except:
136 pass
137 tempprop = tempfile.gettempdir()
138 targets = [omerotemp, homeprop, tempprop]
139
140 name = None
141 choice = None
142 locktest = None
143
144 for target in targets:
145
146 if target is None:
147 continue
148
149 if choice is not None:
150 break
151
152 try:
153
154
155 omero_dir = path(target) / "omero"
156 if omero_dir.exists() and not omero_dir.isdir():
157 self.logger.debug(""""omero" is not a directory: %s""" % omero_dir)
158 continue
159 tmp_dir = omero_dir / "tmp"
160 if tmp_dir.exists() and not tmp_dir.isdir():
161 self.logger.debug(""""tmp" is not a directory: %s""" % tmp_dir)
162 continue
163
164 try:
165
166 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target)
167 locktest = open(name, "a+")
168 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB)
169 locktest.close()
170 locktest = None
171 choice = target
172 self.logger.debug("Chose gloabl tmpdir: %s", choice)
173 finally:
174 if locktest is not None:
175 try:
176 locktest.close()
177 except:
178 self.logger.warn("Failed to close locktest: %s", name, exc_info = True)
179
180 if name is not None:
181 try:
182 os.remove(name)
183 except:
184 self.logger.debug("Failed os.remove(%s)", name)
185
186
187 except exceptions.Exception, e:
188 if "Operation not permitted" in str(e) or \
189 "Operation not supported" in str(e):
190
191
192
193
194 self.logger.debug("%s does not support locking.", target)
195 else:
196 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True)
197
198 if choice is None:
199 raise exceptions.Exception("Could not find lockable tmp dir")
200
201 return path(choice) / "omero" / "tmp"
202
204 """
205 Returns the current OS-user's name
206 """
207 return get_user("Unknown")
208
210 """
211 Returns some representation of the current process's id
212 """
213 return str(os.getpid())
214
216 """
217 Returns True if the current user can write to the given directory
218 """
219 dir = str(dir)
220 return os.access(dir, os.W_OK)
221
223 """
224 If the given directory doesn't exist, creates it (with mode 0700) and returns True.
225 Otherwise False.
226 """
227 dir = path(dir)
228 if not dir.exists():
229 dir.makedirs(0700)
230 return True
231 return False
232
234 """
235 Returns the directory under which all temporary
236 files and folders will be created.
237 """
238 return self.dir
239
240 - def mkstemp(self, prefix, suffix, dir, text = False):
241 """
242 Similar to tempfile.mkstemp name but immediately closes
243 the file descriptor returned and passes back just the name.
244 This prevents various Windows issues"
245 """
246 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text)
247 self.logger.debug("Added file %s", name)
248 try:
249 os.close(fd)
250 except:
251 self.logger.warn("Failed to close fd %s" % fd)
252 return name
253
254 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
255 """
256 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary
257 folders and files, respectively, under self.dir
258 """
259
260 if folder:
261 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir)
262 self.logger.debug("Added folder %s", name)
263 else:
264 name = self.mkstemp(prefix, suffix, self.dir, text)
265
266 return path(name)
267
269 """
270 If the given path is under self.dir, then it is deleted
271 whether file or folder. Otherwise, an exception is thrown.
272 """
273 p = path(name)
274 parpath = p.parpath(self.dir)
275 if len(parpath) < 1:
276 raise exceptions.Exception("%s is not in %s" % (p, self.dir))
277
278 if p.exists():
279 if p.isdir():
280 p.rmtree(onerror = self.on_rmtree)
281 self.logger.debug("Removed folder %s", name)
282 else:
283 p.remove()
284 self.logger.debug("Removed file %s", name)
285
294
296 """
297 Attempts to delete all directories under self.userdir
298 other than the one owned by this process. If a directory
299 is locked, it is skipped.
300 """
301 self.logger.debug("Cleaning user dir: %s" % self.userdir)
302 dirs = self.userdir.dirs()
303 for dir in dirs:
304 if str(dir) == str(self.dir):
305 self.logger.debug("Skipping self: %s", dir)
306 continue
307 lock = dir / ".lock"
308 if lock.exists():
309 f = open(str(lock),"r")
310 try:
311 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB)
312 f.close()
313 except:
314 print "Locked: %s" % dir
315 continue
316 dir.rmtree(onerror=self.on_rmtree)
317 print "Deleted: %s" % dir
318
321
322 manager = TempFileManager()
323 """
324 Global TempFileManager instance for use by the current process and
325 registered with the atexit module for cleaning up all created files on exit.
326 Other instances can be created for specialized purposes.
327 """
328
329 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
330 """
331 Uses the global TempFileManager to create a temporary file.
332 """
333 return manager.create_path(prefix, suffix, folder = folder)
334
336 """
337 Removes the file from the global TempFileManager. The file will be deleted
338 if it still exists.
339 """
340 return manager.remove_path(file)
341
343 """
344 Returns the dir value for the global TempFileManager.
345 """
346 return manager.gettempdir()
347
348 if __name__ == "__main__":
349
350 from omero.util import configure_logging
351
352 if len(sys.argv) > 1:
353 args = sys.argv[1:]
354 else:
355 args = []
356
357
358 if "--debug" in args and "DEBUG" not in os.environ:
359 configure_logging(loglevel=logging.DEBUG)
360 else:
361 configure_logging()
362
363 if "clean" in args:
364 manager.clean_userdir()
365 sys.exit(0)
366 elif "dir" in args:
367 print manager.gettempdir()
368 sys.exit(0)
369 elif "lock" in args:
370 print "Locking %s" % manager.gettempdir()
371 raw_input("Waiting on user input...")
372 sys.exit(0)
373
374 print "Usage: %s clean" % sys.argv[0]
375 print " or: %s dir " % sys.argv[0]
376 sys.exit(2)
377