1
2
3
4
5
6
7 """
8 OMERO Support for temporary files and directories
9 """
10
11 import os
12 import sys
13 import atexit
14 import logging
15 import tempfile
16 import threading
17 import traceback
18 import portalocker
19
20 from path import path
21 from omero.util import get_user_dir, get_user
22
23
24 if "DEBUG" in os.environ:
25 from omero.util import configure_logging
26 configure_logging(loglevel=logging.DEBUG)
27
28
29
30
31
32
33
35 """
36 Creates temporary files and folders and makes a best effort
37 to remove them on exit (or sooner). Typically only a single
38 instance of this class will exist ("manager" variable in this
39 module below)
40 """
41
43 """
44 Initializes a TempFileManager instance with a userDir containing
45 the given prefix value, or "omero" by default. Also registers
46 an atexit callback to call self.cleanup() on exit.
47 """
48 self.logger = logging.getLogger("omero.util.TempFileManager")
49 self.is_win32 = ( sys.platform == "win32" )
50 self.prefix = prefix
51
52 self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username()))
53 """
54 User-accessible directory of the form $TMPDIR/omero_$USERNAME.
55 If the given directory is not writable, an attempt is made
56 to use an alternative
57 """
58 if not self.create(self.userdir) and not self.access(self.userdir):
59 i = 0
60 while i < 10:
61 t = path("%s_%s" % (self.userdir, i))
62 if self.create(t) or self.access(t):
63 self.userdir = t
64 break
65 raise Exception("Failed to create temporary directory: %s" % self.userdir)
66 self.dir = self.userdir / self.pid()
67 """
68 Directory under which all temporary files and folders will be created.
69 An attempt to remove a path not in this directory will lead to an
70 exception.
71 """
72
73
74
75 if not self.dir.exists():
76 self.dir.makedirs()
77 self.logger.debug("Using temp dir: %s" % self.dir)
78
79 self.lock = None
80 try:
81 self.lock = open(str(self.dir / ".lock"), "a+")
82 """
83 .lock file under self.dir which is used to prevent other
84 TempFileManager instances (also in other languages) from
85 cleaning up this directory.
86 """
87 try:
88 portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB)
89 atexit.register(self.cleanup)
90 except:
91 lock = self.lock
92 self.lock = None
93 if lock:
94 self.lock.close()
95 raise
96 finally:
97 try:
98 if not self.lock:
99 self.cleanup()
100 except:
101 self.logger.warn("Error on cleanup after error", exc_info = True)
102
104 """
105 Releases self.lock and deletes self.dir.
106 The lock is released first since on some platforms like Windows
107 the lock file cannot be deleted even by the owner of the lock.
108 """
109 try:
110 if self.lock:
111 self.lock.close()
112 except:
113 self.logger.error("Failed to release lock", exc_info = True)
114 self.clean_tempdir()
115
117 """
118 Returns a platform-specific user-writable temporary directory
119
120 First, the value of "OMERO_TEMPDIR" is attempted (if available),
121 then user's home directory, then the global temp director.
122
123 Typical errors for any of the possible temp locations are:
124 * non-existence
125 * inability to lock
126
127 See: http://trac.openmicroscopy.org.uk/ome/ticket/1653
128 """
129 locktest = None
130
131 omerotemp = os.environ.get("OMERO_TEMPDIR", None)
132 homeprop = None
133 try:
134 homeprop = get_user_dir()
135 except:
136 pass
137 tempprop = tempfile.gettempdir()
138 targets = [omerotemp, homeprop, tempprop]
139
140 name = None
141 choice = None
142 locktest = None
143
144 for target in targets:
145
146 if target is None:
147 continue
148
149 if choice is not None:
150 break
151
152 try:
153
154
155 omero_dir = path(target) / "omero"
156 if omero_dir.exists() and not omero_dir.isdir():
157 self.logger.debug(""""omero" is not a directory: %s""" % omero_dir)
158 continue
159 tmp_dir = omero_dir / "tmp"
160 if tmp_dir.exists() and not tmp_dir.isdir():
161 self.logger.debug(""""tmp" is not a directory: %s""" % tmp_dir)
162 continue
163
164 try:
165
166 name = self.mkstemp(prefix=".lock_test", suffix=".tmp", dir=target)
167 locktest = open(name, "a+")
168 portalocker.lock(locktest, portalocker.LOCK_EX|portalocker.LOCK_NB)
169 locktest.close()
170 locktest = None
171 choice = target
172 self.logger.debug("Chose gloabl tmpdir: %s", choice)
173 finally:
174 if locktest is not None:
175 try:
176 locktest.close()
177 except:
178 self.logger.warn("Failed to close locktest: %s", name, exc_info = True)
179
180 if name is not None:
181 try:
182 os.remove(name)
183 except:
184 self.logger.debug("Failed os.remove(%s)", name)
185
186
187 except Exception, e:
188 if "Operation not permitted" in str(e) or \
189 "Operation not supported" in str(e):
190
191
192
193
194 self.logger.debug("%s does not support locking.", target)
195 else:
196 self.logger.warn("Invalid tmp dir: %s" % target, exc_info = True)
197
198 if choice is None:
199 raise Exception("Could not find lockable tmp dir")
200
201 return path(choice) / "omero" / "tmp"
202
204 """
205 Returns the current OS-user's name
206 """
207 return get_user("Unknown")
208
210 """
211 Returns some representation of the current process's id
212 """
213 return str(os.getpid())
214
216 """
217 Returns True if the current user can write to the given directory
218 """
219 dir = str(dir)
220 return os.access(dir, os.W_OK)
221
223 """
224 If the given directory doesn't exist, creates it (with mode 0700) and returns True.
225 Otherwise False.
226 """
227 dir = path(dir)
228 if not dir.exists():
229 dir.makedirs(0700)
230 return True
231 return False
232
234 """
235 Returns the directory under which all temporary
236 files and folders will be created.
237 """
238 return self.dir
239
240 - def mkstemp(self, prefix, suffix, dir, text = False):
241 """
242 Similar to tempfile.mkstemp name but immediately closes
243 the file descriptor returned and passes back just the name.
244 This prevents various Windows issues"
245 """
246 fd, name = tempfile.mkstemp(prefix = prefix, suffix = suffix, dir = dir, text = text)
247 self.logger.debug("Added file %s", name)
248 try:
249 os.close(fd)
250 except:
251 self.logger.warn("Failed to close fd %s" % fd)
252 return name
253
254 - def create_path(self, prefix, suffix, folder = False, text = False, mode = "r+"):
255 """
256 Uses tempfile.mkdtemp and tempfile.mkstemp to create temporary
257 folders and files, respectively, under self.dir
258 """
259
260 if folder:
261 name = tempfile.mkdtemp(prefix = prefix, suffix = suffix, dir = self.dir)
262 self.logger.debug("Added folder %s", name)
263 else:
264 name = self.mkstemp(prefix, suffix, self.dir, text)
265
266 return path(name)
267
269 """
270 If the given path is under self.dir, then it is deleted
271 whether file or folder. Otherwise, an exception is thrown.
272 """
273 p = path(name)
274 parpath = p.parpath(self.dir)
275 if len(parpath) < 1:
276 raise Exception("%s is not in %s" % (p, self.dir))
277
278 if p.exists():
279 if p.isdir():
280 try:
281 p.rmtree(onerror = self.on_rmtree)
282 self.logger.debug("Removed folder %s", name)
283 except:
284 self.logger.error("Failed to remove folder %s", name)
285 else:
286 try:
287 p.remove()
288 self.logger.debug("Removed file %s", name)
289 except:
290 self.logger.error("Failed to remove file %s", name)
291
300
302 """
303 Attempts to delete all directories under self.userdir
304 other than the one owned by this process. If a directory
305 is locked, it is skipped.
306 """
307 self.logger.debug("Cleaning user dir: %s" % self.userdir)
308 dirs = self.userdir.dirs()
309 for dir in dirs:
310 if str(dir) == str(self.dir):
311 self.logger.debug("Skipping self: %s", dir)
312 continue
313 lock = dir / ".lock"
314 if lock.exists():
315 f = open(str(lock),"r")
316 try:
317 portalocker.lock(f, portalocker.LOCK_EX|portalocker.LOCK_NB)
318 f.close()
319 except:
320 print "Locked: %s" % dir
321 continue
322 dir.rmtree(onerror=self.on_rmtree)
323 print "Deleted: %s" % dir
324
327
328 manager = TempFileManager()
329 """
330 Global TempFileManager instance for use by the current process and
331 registered with the atexit module for cleaning up all created files on exit.
332 Other instances can be created for specialized purposes.
333 """
334
335 -def create_path(prefix = "omero", suffix = ".tmp", folder = False):
336 """
337 Uses the global TempFileManager to create a temporary file.
338 """
339 return manager.create_path(prefix, suffix, folder = folder)
340
342 """
343 Removes the file from the global TempFileManager. The file will be deleted
344 if it still exists.
345 """
346 return manager.remove_path(file)
347
349 """
350 Returns the dir value for the global TempFileManager.
351 """
352 return manager.gettempdir()
353
354 if __name__ == "__main__":
355
356 from omero.util import configure_logging
357
358 if len(sys.argv) > 1:
359 args = sys.argv[1:]
360 else:
361 args = []
362
363
364 if "--debug" in args and "DEBUG" not in os.environ:
365 configure_logging(loglevel=logging.DEBUG)
366 else:
367 configure_logging()
368
369 if "clean" in args:
370 manager.clean_userdir()
371 sys.exit(0)
372 elif "dir" in args:
373 print manager.gettempdir()
374 sys.exit(0)
375 elif "lock" in args:
376 print "Locking %s" % manager.gettempdir()
377 raw_input("Waiting on user input...")
378 sys.exit(0)
379
380 print "Usage: %s clean" % sys.argv[0]
381 print " or: %s dir " % sys.argv[0]
382 sys.exit(2)
383