1
2
3
4
5
6
7
8 import os
9 import sys
10 import time
11 import signal
12 import logging
13 import traceback
14 import exceptions
15 import killableprocess as subprocess
16
17 from path import path
18
19 import Ice
20 import omero
21 import omero.clients
22 import omero.util
23 import omero.util.concurrency
24
25 from omero.util.temp_files import create_path, remove_path
26 from omero.util.decorators import remoted, perf, locked
27 from omero.rtypes import *
28
29
30 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
31 """
32 Wrapper around a subprocess.Popen instance. Returned by ProcessorI
33 when a job is submitted. This implementation uses the given
34 interpreter to call a file that must be named "script" in the
35 generated temporary directory.
36
37 Call is equivalent to:
38
39 cd TMP_DIR
40 ICE_CONFIG=./config interpreter ./script >out 2>err &
41
42 The properties argument is used to generate the ./config file.
43
44 The params argument may be null in which case this process
45 is being used solely to calculate the parameters for the script
46 ("omero.scripts.parse=true")
47
48 If iskill is True, then on cleanup, this process will reap the
49 attached session completely.
50 """
51
52 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\
53 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast):
54 """
55 Popen and callback_Cast are primarily for testing.
56 """
57 omero.util.SimpleServant.__init__(self, ctx)
58 self.interpreter = interpreter
59 self.properties = properties
60 self.params = params
61 self.iskill = iskill
62 self.Popen = Popen
63 self.callback_cast = callback_cast
64
65 self.rcode = None
66 self.callbacks = {}
67 self.popen = None
68 self.pid = None
69 self.started = None
70 self.stopped = None
71
72 self.uuid = properties["omero.user"]
73
74
75 self.make_files()
76 self.make_env()
77 self.make_config()
78 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
79
80
81
82
83
85 self.env = omero.util.Environment("PATH", "PYTHONPATH",\
86 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR")
87
88
89
90
91
92
93
94 self.env.append("PYTHONPATH", str(path.getcwd() / "lib" / "python"))
95 self.env.set("ICE_CONFIG", str(self.config_path))
96
98 self.dir = create_path("process", ".dir", folder = True)
99 self.script_path = self.dir / "script"
100 self.config_path = self.dir / "config"
101 self.stdout_path = self.dir / "out"
102 self.stderr_path = self.dir / "err"
103
105 """
106 Creates the ICE_CONFIG file used by the client.
107 """
108 config_file = open(str(self.config_path), "w")
109 try:
110 for key in self.properties.iterkeys():
111 config_file.write("%s=%s\n"%(key, self.properties[key]))
112 finally:
113 config_file.close()
114
116 """
117 Create a client for performing cleanup operations.
118 This client should be closed as soon as possible
119 by the process
120 """
121 try:
122 client = omero.client(["--Ice.Config=%s" % str(self.config_path)])
123 client.createSession().detachOnDestroy()
124 self.logger.debug("client: %s" % client.sf)
125 return client
126 except:
127 self.logger.error("Failed to create client for %s" % self.uuid)
128 return None
129
130
131
132
133
134 @locked
136 """
137 Process creation has to wait until all external downloads, etc
138 are finished.
139 """
140
141 if self.isActive():
142 raise omero.ApiUsageException(None, None, "Already activated")
143
144 self.stdout = open(str(self.stdout_path), "w")
145 self.stderr = open(str(self.stderr_path), "w")
146 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr)
147 self.pid = self.popen.pid
148 self.started = time.time()
149 self.stopped = None
150 self.status("Activated")
151
152 @locked
154 """
155 Cleans up the temporary directory used by the process, and terminates
156 the Popen process if running.
157 """
158
159 if not self.isActive():
160 raise omero.ApiUsageException(None, None, "Not active")
161
162 if self.stopped:
163
164 return
165
166 self.stopped = time.time()
167 d_start = time.time()
168 self.status("Deactivating")
169
170
171 try:
172
173 self.shutdown()
174 self.popen = None
175
176 self.cleanup_output()
177 self.upload_output()
178 self.cleanup_tmpdir()
179
180 except exceptions.Exception:
181 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True)
182
183 d_stop = time.time()
184 elapsed = int(self.stopped - self.started)
185 d_elapsed = int(d_stop - d_start)
186 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
187
188 @locked
190 """
191 Tests only if this instance has a non-None popen attribute. After activation
192 this method will return True until the popen itself returns a non-None
193 value (self.rcode) at which time it will be nulled and this method will again
194 return False
195 """
196 return self.popen is not None
197
198 @locked
200 """
201 Returns true only if this instance has either a non-null
202 popen or a non-null rcode field.
203 """
204 return self.popen is not None or self.rcode is not None
205
206 @locked
208 return self.popen is not None and self.rcode is None
209
210 @locked
212 return self.rcode is not None
213
214 @locked
216 """
217 Allows short-cutting various checks if we already
218 have a rcode for this popen. A non-None return value
219 implies that a process was started and returned
220 the given non-None value itself.
221 """
222 if not self.wasActivated:
223 raise omero.InternalException(None, None, "Process never activated")
224 return self.isFinished()
225
226
227
228
229
232
233 @perf
234 @locked
236 """
237 Called periodically to keep the session alive. Returns
238 False if this resource can be cleaned up. (Resources API)
239 """
240
241 if not self.wasActivated():
242 return True
243
244 try:
245 self.poll()
246 self.ctx.getSession().getSessionService().getSession(self.uuid)
247 return True
248 except:
249 self.status("Keep alive failed")
250 return False
251
252 @perf
253 @locked
255 """
256 Deactivates the process (if active) and cleanups the server
257 connection. (Resources API)
258 """
259
260 if self.isRunning():
261 self.deactivate()
262
263 try:
264 sf = self.ctx.getSession(recreate = False)
265 except:
266 self.logger.warn("Can't get session for cleanup")
267 return
268
269 self.status("Cleaning")
270 svc = sf.getSessionService()
271 obj = omero.model.SessionI()
272 obj.uuid = omero.rtypes.rstring(self.uuid)
273 try:
274 if self.iskill:
275 self.status("Killing session")
276 while svc.closeSession(obj) > 0:
277 pass
278
279
280 except:
281 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
282
284 """
285 Flush and close the stderr and stdout streams.
286 """
287 try:
288 if hasattr(self, "stderr"):
289 self.stderr.flush()
290 self.stderr.close()
291 except:
292 self.logger.error("cleanup of sterr failed", exc_info = True)
293 try:
294 if hasattr(self, "stdout"):
295 self.stdout.flush()
296 self.stdout.close()
297 except:
298 self.logger.error("cleanup of sterr failed", exc_info = True)
299
301 """
302 If this is not a params calculation (i.e. parms != null) and the
303 stdout or stderr are non-null, they they will be uploaded and
304 attached to the job.
305 """
306 client = self.tmp_client()
307 if not client:
308 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid)
309 return
310
311 try:
312 if self.params:
313 out_format = self.params.stdoutFormat
314 err_format = self.params.stderrFormat
315 upload = True
316 else:
317 out_format = "text/plain"
318 err_format = out_format
319 upload = False
320
321 self._upload(upload, client, self.stdout_path, "stdout", out_format)
322 self._upload(upload, client, self.stderr_path, "stderr", err_format)
323 finally:
324 client.__del__()
325
326 - def _upload(self, upload, client, filename, name, format):
327
328 if not upload or not format:
329 return
330
331 filename = str(filename)
332 sz = os.path.getsize(filename)
333 if not sz:
334 self.logger.info("No %s" % name)
335 return
336
337 try:
338 ofile = client.upload(filename, name=name, type=format)
339 jobid = long(client.getProperty("omero.job"))
340 link = omero.model.JobOriginalFileLinkI()
341 link.parent = omero.model.ScriptJobI(rlong(jobid), False)
342 link.child = ofile
343 client.getSession().getUpdateService().saveObject(link)
344 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val))
345 except:
346 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
347
349 """
350 Remove all known files and finally the temporary directory.
351 If other files exist, an exception will be raised.
352 """
353 try:
354 remove_path(self.dir)
355 except:
356 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
357
358
359
360
361
366
367 @perf
368 @remoted
369 - def poll(self, current = None):
370 """
371 Checks popen.poll() (if active) and notifies all callbacks
372 if necessary. If this method returns a non-None value, then
373 the process will be marked inactive.
374 """
375
376 if self.alreadyDone():
377 return rint(self.rcode)
378
379 self.status("Polling")
380 if self.rcode is None:
381
382 return None
383 else:
384 self.deactivate()
385 rv = rint(self.rcode)
386 self.allcallbacks("processFinished", rv)
387 return rv
388
389 @perf
390 @remoted
391 - def wait(self, current = None):
392 """
393 Waits on popen.wait() to return (if active) and notifies
394 all callbacks. Marks this process as inactive.
395 """
396
397 if self.alreadyDone():
398 return self.rcode
399
400 self.status("Waiting")
401 self.rcode = self.popen.wait()
402 self.deactivate()
403 self.allcallbacks("processFinished", self.rcode)
404 return self.rcode
405
407 """
408 Attempts to cancel the process by sending SIGTERM
409 (or similar)
410 """
411 try:
412 self.status("os.kill(TERM)")
413 os.kill(self.popen.pid, signal.SIGTERM)
414 except AttributeError:
415 self.logger.debug("No os.kill(TERM). Skipping cancel")
416
417 - def _send(self, iskill):
418 """
419 Helper method for sending signals. This method only
420 makes a call is the process is active.
421 """
422 if self.isRunning():
423 try:
424 if self.popen.poll() is None:
425 if iskill:
426 self.status("popen.kill(True)")
427 self.popen.kill(True)
428 else:
429 self._term()
430
431 else:
432 self.status("Skipped signal")
433 except OSError, oserr:
434 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
435
436 @perf
437 @remoted
438 - def cancel(self, current = None):
439 """
440 Tries to cancel popen (if active) and notifies callbacks.
441 """
442
443 if self.alreadyDone():
444 return True
445
446 self._send(iskill=False)
447 finished = self.isFinished()
448 if finished:
449 self.deactivate()
450 self.allcallbacks("processCancelled", finished)
451 return finished
452
453 @perf
454 @remoted
455 - def kill(self, current = None):
466
467 @perf
468 @remoted
470 """
471 If self.popen is active, then first call cancel, wait a period of
472 time, and finally call kill.
473 """
474
475 if self.alreadyDone():
476 return
477
478 self.status("Shutdown")
479 try:
480 for i in range(5, 0, -1):
481 if self.cancel():
482 break
483 else:
484 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1)
485 self.stop_event.wait(6)
486 self.kill()
487 except:
488 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
489
490
491
492
493
494 @remoted
495 @locked
497 try:
498 id = callback.ice_getIdentity()
499 key = "%s/%s" % (id.category, id.name)
500 callback = callback.ice_oneway()
501 callback = self.callback_cast(callback)
502 if not callback:
503 e = "Callback is invalid"
504 else:
505 self.callbacks[key] = callback
506 self.logger.debug("Added callback: %s", key)
507 return
508 except exceptions.Exception, ex:
509 e = ex
510
511 msg = "Failed to add callback: %s. Reason: %s" % (callback, e)
512 self.logger.debug(msg)
513 raise omero.ApiUsageException(None, None, msg)
514
515 @remoted
516 @locked
518 try:
519 id = callback.ice_getIdentity()
520 key = "%s/%s" % (id.category, id.name)
521 if not key in self.callback:
522 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key)
523 del self.callbacks[key]
524 self.logger.debug("Removed callback: %s", key)
525 except exceptions.Exception, e:
526 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e)
527 self.logger.debug(msg)
528 raise omero.ApiUsageException(None, None, msg)
529
530 @locked
532 self.status("Callback %s" % method)
533 for key, cb in self.callbacks.items():
534 try:
535 m = getattr(cb, method)
536 m(arg)
537 except:
538 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
539
541 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, self.rcode, self.uuid)
542
543 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
544
549
550 @remoted
551 - def parseJob(self, session, job, current = None):
560
561 @remoted
562 - def processJob(self, session, job, current = None):
573
574 @perf
575 - def parse(self, client, session, job, current, iskill):
576
577 if not session or not job or not job.id:
578 raise omero.ApiUsageException("No null arguments")
579
580 self.logger.info("parseJob: Session = %s, JobId = %s" % (session, job.id.val))
581 properties = {}
582 properties["omero.scripts.parse"] = "true"
583 process = self.process(client, session, job, current, None, properties, iskill)
584 process.wait()
585 rv = client.getOutput("omero.scripts.parse")
586 if rv != None:
587 return rv.val
588 else:
589 self.logger.warning("No output found for omero.scripts.parse. Keys: %s" % client.getOutputKeys())
590 return None
591
592 @perf
593 - def process(self, client, session, job, current, params, properties = {}, iskill = True):
594 """
595 session: session uuid, used primarily if client is None
596 client: an omero.client object which should be attached to a session
597 """
598
599 sf = client.getSession()
600 handle = sf.createJobHandle()
601 handle.attach(job.id.val)
602 if handle.jobFinished():
603 raise omero.ApiUsageException("Job already finished.")
604
605 file = sf.getQueryService().findByQuery(\
606 """select o from Job j
607 join j.originalFileLinks links
608 join links.child o
609 join o.format
610 where
611 j.id = %d
612 and o.details.owner.id = 0
613 and o.format.value = 'text/x-python'
614 """ % job.id.val, None)
615
616
617 if not file:
618 raise omero.ApiUsageException(\
619 None, None, "Job should have one executable file attached.")
620
621 properties["omero.job"] = str(job.id.val)
622 properties["omero.user"] = session
623 properties["omero.pass"] = session
624 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router")
625
626 self.logger.info("processJob: Session = %s, JobId = %s" % (session, job.id.val))
627 process = ProcessI(self.ctx, "python", properties, params, iskill)
628 self.resources.add(process)
629 client.download(file, str(process.script_path))
630 self.logger.info("Downloaded file: %s" % file.id.val)
631 s = client.sha1(str(process.script_path))
632 if not s == file.sha1.val:
633 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s)
634 self.logger.error(msg)
635 process.cleanup()
636 raise omero.InternalException(None, None, msg)
637 else:
638 process.activate()
639
640 prx = current.adapter.addWithUUID(process)
641 return omero.grid.ProcessPrx.uncheckedCast(prx)
642