1
2
3
4
5
6
7
8 import os
9 import sys
10 import time
11 import signal
12 import logging
13 import traceback
14 import exceptions
15 import killableprocess as subprocess
16
17 from path import path
18
19 import Ice
20 import omero
21 import omero.clients
22 import omero.scripts
23 import omero.util
24 import omero.util.concurrency
25
26 from omero.util.temp_files import create_path, remove_path
27 from omero.util.decorators import remoted, perf, locked
28 from omero.rtypes import *
29 from omero.util.decorators import remoted, perf, wraps
30
31 -def with_context(func, context):
32 """ Decorator for invoking Ice methods with a context """
33 def handler(*args, **kwargs):
34 args = list(args)
35 args.append(context)
36 return func(*args, **kwargs)
37 handler = wraps(func)(handler)
38 return handler
39
41 """
42 Wraps a ServiceInterfacePrx instance and applies
43 a "omero.group" to the passed context on every
44 invotation.
45
46 For example, using a job handle as root requires logging
47 manually into the group. (ticket:2044)
48 """
49
51 self._service = service
52 self._group_id = str(group_id)
53
55 ctx = self._service.ice_getCommunicator().getImplicitContext().getContext()
56 ctx = dict(ctx)
57 ctx["omero.group"] = group
58 return ctx
59
61 if name.startswith("_"):
62 return self.__dict__[name]
63 elif hasattr(self._service, name):
64 method = getattr(self._service, name)
65 ctx = self._get_ctx(self._group_id)
66 return with_context(method, ctx)
67 raise AttributeError("'%s' object has no attribute '%s'" % (self.service, name))
68
69 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
70 """
71 Wrapper around a subprocess.Popen instance. Returned by ProcessorI
72 when a job is submitted. This implementation uses the given
73 interpreter to call a file that must be named "script" in the
74 generated temporary directory.
75
76 Call is equivalent to:
77
78 cd TMP_DIR
79 ICE_CONFIG=./config interpreter ./script >out 2>err &
80
81 The properties argument is used to generate the ./config file.
82
83 The params argument may be null in which case this process
84 is being used solely to calculate the parameters for the script
85 ("omero.scripts.parse=true")
86
87 If iskill is True, then on cleanup, this process will reap the
88 attached session completely.
89 """
90
91 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\
92 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast,\
93 omero_home = path.getcwd()):
94 """
95 Popen and callback_Cast are primarily for testing.
96 """
97 omero.util.SimpleServant.__init__(self, ctx)
98 self.omero_home = omero_home
99 self.interpreter = interpreter
100 self.properties = properties
101 self.params = params
102 self.iskill = iskill
103 self.Popen = Popen
104 self.callback_cast = callback_cast
105
106 self.rcode = None
107 self.callbacks = {}
108 self.popen = None
109 self.pid = None
110 self.started = None
111 self.stopped = None
112 self.final_status = None
113
114 self.uuid = properties["omero.user"]
115
116
117 self.make_files()
118 self.make_env()
119 self.make_config()
120 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
121
122
123
124
125
127 self.env = omero.util.Environment("PATH", "PYTHONPATH",\
128 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR", "HOME")
129
130
131
132
133
134
135
136 self.env.append("PYTHONPATH", str(self.omero_home / "lib" / "python"))
137 self.env.set("ICE_CONFIG", str(self.config_path))
138
140 self.dir = create_path("process", ".dir", folder = True)
141 self.script_path = self.dir / "script"
142 self.config_path = self.dir / "config"
143 self.stdout_path = self.dir / "out"
144 self.stderr_path = self.dir / "err"
145
147 """
148 Creates the ICE_CONFIG file used by the client.
149 """
150 config_file = open(str(self.config_path), "w")
151 try:
152 for key in self.properties.iterkeys():
153 config_file.write("%s=%s\n"%(key, self.properties[key]))
154 finally:
155 config_file.close()
156
158 """
159 Create a client for performing cleanup operations.
160 This client should be closed as soon as possible
161 by the process
162 """
163 try:
164 client = omero.client(["--Ice.Config=%s" % str(self.config_path)])
165 client.setAgent("OMERO.process")
166 client.createSession().detachOnDestroy()
167 self.logger.debug("client: %s" % client.sf)
168 return client
169 except:
170 self.logger.error("Failed to create client for %s" % self.uuid)
171 return None
172
173
174
175
176
177 @locked
179 """
180 Process creation has to wait until all external downloads, etc
181 are finished.
182 """
183
184 if self.isActive():
185 raise omero.ApiUsageException(None, None, "Already activated")
186
187 self.stdout = open(str(self.stdout_path), "w")
188 self.stderr = open(str(self.stderr_path), "w")
189 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr)
190 self.pid = self.popen.pid
191 self.started = time.time()
192 self.stopped = None
193 self.status("Activated")
194
195 @locked
197 """
198 Cleans up the temporary directory used by the process, and terminates
199 the Popen process if running.
200 """
201
202 if not self.isActive():
203 raise omero.ApiUsageException(None, None, "Not active")
204
205 if self.stopped:
206
207 return
208
209 self.stopped = time.time()
210 d_start = time.time()
211 self.status("Deactivating")
212
213
214 try:
215
216 self.shutdown()
217 self.popen = None
218
219 client = self.tmp_client()
220 try:
221 self.set_job_status(client)
222 self.cleanup_output()
223 self.upload_output(client)
224 self.cleanup_tmpdir()
225 finally:
226 if client:
227 client.__del__()
228
229 except exceptions.Exception:
230 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True)
231
232 d_stop = time.time()
233 elapsed = int(self.stopped - self.started)
234 d_elapsed = int(d_stop - d_start)
235 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
236
237 @locked
239 """
240 Tests only if this instance has a non-None popen attribute. After activation
241 this method will return True until the popen itself returns a non-None
242 value (self.rcode) at which time it will be nulled and this method will again
243 return False
244 """
245 return self.popen is not None
246
247 @locked
249 """
250 Returns true only if this instance has either a non-null
251 popen or a non-null rcode field.
252 """
253 return self.popen is not None or self.rcode is not None
254
255 @locked
257 return self.popen is not None and self.rcode is None
258
259 @locked
261 return self.rcode is not None
262
263 @locked
265 """
266 Allows short-cutting various checks if we already
267 have a rcode for this popen. A non-None return value
268 implies that a process was started and returned
269 the given non-None value itself.
270 """
271 if not self.wasActivated:
272 raise omero.InternalException(None, None, "Process never activated")
273 return self.isFinished()
274
275
276
277
278
281
282 @perf
283 @locked
285 """
286 Called periodically to keep the session alive. Returns
287 False if this resource can be cleaned up. (Resources API)
288 """
289
290 if not self.wasActivated():
291 return True
292
293 try:
294 self.poll()
295 self.ctx.getSession().getSessionService().getSession(self.uuid)
296 return True
297 except:
298 self.status("Keep alive failed")
299 return False
300
301 @perf
302 @locked
304 """
305 Deactivates the process (if active) and cleanups the server
306 connection. (Resources API)
307 """
308
309 if self.isRunning():
310 self.deactivate()
311
312 if not self.iskill:
313 return
314
315 try:
316 sf = self.ctx.getSession(recreate = False)
317 except:
318 self.logger.debug("Can't get session for cleanup")
319 return
320
321 self.status("Killing session")
322 svc = sf.getSessionService()
323 obj = omero.model.SessionI()
324 obj.uuid = omero.rtypes.rstring(self.uuid)
325 try:
326 while svc.closeSession(obj) > 0:
327 pass
328
329
330 except:
331 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
332
334 """
335 Flush and close the stderr and stdout streams.
336 """
337 try:
338 if hasattr(self, "stderr"):
339 self.stderr.flush()
340 self.stderr.close()
341 except:
342 self.logger.error("cleanup of sterr failed", exc_info = True)
343 try:
344 if hasattr(self, "stdout"):
345 self.stdout.flush()
346 self.stdout.close()
347 except:
348 self.logger.error("cleanup of sterr failed", exc_info = True)
349
369
371 """
372 If this is not a params calculation (i.e. parms != null) and the
373 stdout or stderr are non-null, they they will be uploaded and
374 attached to the job.
375 """
376 if not client:
377 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid)
378 return
379
380 if self.params:
381 out_format = self.params.stdoutFormat
382 err_format = self.params.stderrFormat
383 else:
384 out_format = "text/plain"
385 err_format = out_format
386
387 self._upload(client, self.stdout_path, "stdout", out_format)
388 self._upload(client, self.stderr_path, "stderr", err_format)
389
390 - def _upload(self, client, filename, name, format):
391
392 if not format:
393 return
394
395 filename = str(filename)
396 sz = os.path.getsize(filename)
397 if not sz:
398 self.status("No %s" % name)
399 return
400
401 try:
402 ofile = client.upload(filename, name=name, type=format)
403 jobid = long(client.getProperty("omero.job"))
404 link = omero.model.JobOriginalFileLinkI()
405 if self.params is None:
406 link.parent = omero.model.ParseJobI(rlong(jobid), False)
407 else:
408 link.parent = omero.model.ScriptJobI(rlong(jobid), False)
409 link.child = ofile
410 client.getSession().getUpdateService().saveObject(link)
411 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val))
412 except:
413 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
414
416 """
417 Remove all known files and finally the temporary directory.
418 If other files exist, an exception will be raised.
419 """
420 try:
421 remove_path(self.dir)
422 except:
423 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
424
425
426
427
428
433
434 @perf
435 @remoted
436 - def poll(self, current = None):
437 """
438 Checks popen.poll() (if active) and notifies all callbacks
439 if necessary. If this method returns a non-None value, then
440 the process will be marked inactive.
441 """
442
443 if self.alreadyDone():
444 return rint(self.rcode)
445
446 self.status("Polling")
447 if self.rcode is None:
448
449 return None
450 else:
451 self.deactivate()
452 rv = rint(self.rcode)
453 self.allcallbacks("processFinished", self.rcode)
454 return rv
455
456 @perf
457 @remoted
458 - def wait(self, current = None):
459 """
460 Waits on popen.wait() to return (if active) and notifies
461 all callbacks. Marks this process as inactive.
462 """
463
464 if self.alreadyDone():
465 return self.rcode
466
467 self.status("Waiting")
468 self.rcode = self.popen.wait()
469 self.deactivate()
470 self.allcallbacks("processFinished", self.rcode)
471 return self.rcode
472
474 """
475 Attempts to cancel the process by sending SIGTERM
476 (or similar)
477 """
478 try:
479 self.status("os.kill(TERM)")
480 os.kill(self.popen.pid, signal.SIGTERM)
481 except AttributeError:
482 self.logger.debug("No os.kill(TERM). Skipping cancel")
483
484 - def _send(self, iskill):
485 """
486 Helper method for sending signals. This method only
487 makes a call is the process is active.
488 """
489 if self.isRunning():
490 try:
491 if self.popen.poll() is None:
492 if iskill:
493 self.status("popen.kill(True)")
494 self.popen.kill(True)
495 else:
496 self._term()
497
498 else:
499 self.status("Skipped signal")
500 except OSError, oserr:
501 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
502
503 @perf
504 @remoted
505 - def cancel(self, current = None):
520
521 @perf
522 @remoted
523 - def kill(self, current = None):
535
536 @perf
537 @remoted
539 """
540 If self.popen is active, then first call cancel, wait a period of
541 time, and finally call kill.
542 """
543
544 if self.alreadyDone():
545 return
546
547 self.status("Shutdown")
548 try:
549 for i in range(5, 0, -1):
550 if self.cancel():
551 break
552 else:
553 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1)
554 self.stop_event.wait(6)
555 self.kill()
556 except:
557 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
558
559
560
561
562
563 @remoted
564 @locked
566 try:
567 id = callback.ice_getIdentity()
568 key = "%s/%s" % (id.category, id.name)
569 callback = callback.ice_oneway()
570 callback = self.callback_cast(callback)
571 if not callback:
572 e = "Callback is invalid"
573 else:
574 self.callbacks[key] = callback
575 self.logger.debug("Added callback: %s", key)
576 return
577 except exceptions.Exception, ex:
578 e = ex
579
580 msg = "Failed to add callback: %s. Reason: %s" % (callback, e)
581 self.logger.debug(msg)
582 raise omero.ApiUsageException(None, None, msg)
583
584 @remoted
585 @locked
587 try:
588 id = callback.ice_getIdentity()
589 key = "%s/%s" % (id.category, id.name)
590 if not key in self.callback:
591 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key)
592 del self.callbacks[key]
593 self.logger.debug("Removed callback: %s", key)
594 except exceptions.Exception, e:
595 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e)
596 self.logger.debug(msg)
597 raise omero.ApiUsageException(None, None, msg)
598
599 @locked
601 self.status("Callback %s" % method)
602 for key, cb in self.callbacks.items():
603 try:
604 m = getattr(cb, method)
605 m(arg)
606 except Ice.LocalException, e:
607 self.logger.debug("LocalException calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = False)
608 except:
609 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
610
612 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, (self.rcode is None and "-" or self.rcode), self.uuid)
613
615
618
620 try:
621 self.sf.keepAlive(None)
622 return True
623 except:
624 return False
625
628
629 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
630
631 - def __init__(self, ctx, needs_session = True,
632 use_session = None, accepts_list = [], cfg = None,
633 omero_home = path.getcwd()):
634
635 self.omero_home = omero_home
636
637
638
639 self.use_session = use_session
640 """
641 If set, this session will be returned from internal_session and
642 the "needs_session" setting ignored.
643 """
644
645 if self.use_session:
646 needs_session = False
647
648 self.accepts_list = accepts_list
649 """
650 A list of contexts which will be accepted by this user-mode
651 processor.
652 """
653
654 omero.util.Servant.__init__(self, ctx, needs_session = needs_session)
655 if cfg is None:
656 self.cfg = os.path.join(os.curdir, "etc", "ice.config")
657 self.cfg = os.path.abspath(self.cfg)
658 else:
659 self.cfg = cfg
660
661
662 self.resources.add( UseSessionHolder(use_session) )
663
665 """
666 Overrides the default action in order to register this proxy
667 with the session's sharedResources to register for callbacks.
668 The on_newsession handler will also keep new sessions informed.
669
670 See ticket:2304
671 """
672 omero.util.Servant.setProxy(self, prx)
673 session = self.internal_session()
674 self.register_session(session)
675
676
677 self.ctx.on_newsession = self.register_session
678
691
693 """
694 Returns the session which should be used for lookups by this instance.
695 Some methods will create a session based on the session parameter.
696 In these cases, the session will belong to the user who is running a
697 script.
698 """
699 if self.use_session:
700 return self.use_session
701 else:
702 return self.ctx.getSession()
703
705 self.logger.info("Registering processor %s", self.prx)
706 prx = omero.grid.ProcessorPrx.uncheckedCast(self.prx)
707 session.sharedResources().addProcessor(prx)
708
727
728 @remoted
729 - def willAccept(self, userContext, groupContext, scriptContext, cb, current = None):
730
731 userID = None
732 if userContext != None:
733 userID = userContext.id.val
734
735 groupID = None
736 if groupContext != None:
737 groupID = groupContext.id.val
738
739 scriptID = None
740 if scriptContext != None:
741 scriptID = scriptContext.id.val
742
743 if scriptID:
744 try:
745 file, handle = self.lookup(scriptContext)
746 handle.close()
747 valid = (file is not None)
748 except:
749 self.logger.error("File lookup failed: user=%s, group=%s, script=%s",\
750 userID, groupID, scriptID, exc_info=1)
751 return
752 else:
753 valid = False
754 for x in self.accepts_list:
755 if isinstance(x, omero.model.Experimenter) and x.id.val == userID:
756 valid = True
757 elif isinstance(x, omero.model.ExperimenterGroup) and x.id.val == groupID:
758 valid = True
759
760 self.logger.debug("Accepts called on: user:%s group:%s scriptjob:%s - Valid: %s",
761 userID, groupID, scriptID, valid)
762
763 try:
764 id = self.internal_session().ice_getIdentity().name
765 cb = cb.ice_oneway()
766 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb)
767 cb.isAccepted(valid, id, str(self.prx))
768 except exceptions.Exception, e:
769 self.logger.warn("callback failed on willAccept: %s Exception:%s", cb, e)
770
771 return valid
772
773 @remoted
775
776 try:
777 cb = cb.ice_oneway()
778 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb)
779 servants = list(self.ctx.servant_map.values())
780 rv = []
781 for x in servants:
782 if hasattr(x, "properties"):
783 rv.append(long(x))
784 cb.responseRunning(rv)
785 except exceptions.Exception, e:
786 self.logger.warn("callback failed on requestRunning: %s Exception:%s", cb, e)
787
788
789 @remoted
790 - def parseJob(self, session, job, current = None):
810
811 @remoted
812 - def processJob(self, session, params, job, current = None):
824
825
826 @perf
827 - def process(self, client, session, job, current, params, properties = {}, iskill = True):
828 """
829 session: session uuid, used primarily if client is None
830 client: an omero.client object which should be attached to a session
831 """
832
833 if not session or not job or not job.id:
834 raise omero.ApiUsageException("No null arguments")
835
836 file, handle = self.lookup(job)
837
838 try:
839 if not file:
840 raise omero.ApiUsageException(\
841 None, None, "Job should have one executable file attached.")
842
843 sf = self.internal_session()
844 if params:
845 self.logger.debug("Checking params for job %s" % job.id.val)
846 svc = sf.getSessionService()
847 inputs = svc.getInputs(session)
848 errors = omero.scripts.validate_inputs(params, inputs, svc, session)
849 if errors:
850 errors = "Invalid parameters:\n%s" % errors
851 raise omero.ValidationException(None, None, errors)
852
853 properties["omero.job"] = str(job.id.val)
854 properties["omero.user"] = session
855 properties["omero.pass"] = session
856 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router")
857
858 process = ProcessI(self.ctx, "python", properties, params, iskill, omero_home = self.omero_home)
859 self.resources.add(process)
860
861
862 scriptText = sf.getScriptService().getScriptText(file.id.val)
863 process.script_path.write_bytes(scriptText)
864
865 self.logger.info("Downloaded file: %s" % file.id.val)
866 s = client.sha1(str(process.script_path))
867 if not s == file.sha1.val:
868 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s)
869 self.logger.error(msg)
870 process.cleanup()
871 raise omero.InternalException(None, None, msg)
872 else:
873 process.activate()
874 handle.setStatus("Running")
875
876 prx = self.ctx.add_servant(current, process)
877 return omero.grid.ProcessPrx.uncheckedCast(prx), process
878
879 finally:
880 handle.close()
881
882 -def usermode_processor(client, serverid = "UsermodeProcessor",\
883 cfg = None, accepts_list = None, stop_event = None,\
884 omero_home = path.getcwd()):
885 """
886 Creates an activates a usermode processor for the given client.
887 It is the responsibility of the client to call "cleanup()" on
888 the ProcessorI implementation which is returned.
889
890 cfg is the path to an --Ice.Config-valid file or files. If none
891 is given, the value of ICE_CONFIG will be taken from the environment
892 if available. Otherwise, all properties will be taken from the client
893 instance.
894
895 accepts_list is the list of IObject instances which will be passed to
896 omero.api.IScripts.validateScript. If none is given, only the current
897 Experimenter's own object will be passed.
898
899 stop_event is an threading.Event. One will be acquired from
900 omero.util.concurrency.get_event if none is provided.
901 """
902
903 if cfg is None:
904 cfg = os.environ.get("ICE_CONFIG")
905
906 if accepts_list is None:
907 uid = client.sf.getAdminService().getEventContext().userId
908 accepts_list = [omero.model.ExperimenterI(uid, False)]
909
910 if stop_event is None:
911 stop_event = omero.util.concurrency.get_event(name="UsermodeProcessor")
912
913 ctx = omero.util.ServerContext(serverid, client.ic, stop_event)
914 impl = omero.processor.ProcessorI(ctx,
915 use_session=client.sf, accepts_list=accepts_list, cfg=cfg,
916 omero_home = omero_home)
917 ctx.add_servant(client.adapter, impl)
918 return impl
919