1
2
3
4
5
6
7
8 import os
9 import time
10 import signal
11 import logging
12 import traceback
13 import exceptions
14 import killableprocess as subprocess
15
16 from path import path
17
18 import Ice
19 import omero
20 import omero.clients
21 import omero.scripts
22 import omero.util
23 import omero.util.concurrency
24
25 from omero.util.temp_files import create_path, remove_path
26 from omero.util.decorators import remoted, perf, locked
27 from omero.rtypes import *
28 from omero.util.decorators import remoted, perf, wraps
29
30 sys = __import__("sys")
31
32
33 -def with_context(func, context):
34 """ Decorator for invoking Ice methods with a context """
35 def handler(*args, **kwargs):
36 args = list(args)
37 args.append(context)
38 return func(*args, **kwargs)
39 handler = wraps(func)(handler)
40 return handler
41
43 """
44 Wraps a ServiceInterfacePrx instance and applies
45 a "omero.group" to the passed context on every
46 invotation.
47
48 For example, using a job handle as root requires logging
49 manually into the group. (ticket:2044)
50 """
51
53 self._service = service
54 self._group_id = str(group_id)
55
57 ctx = self._service.ice_getCommunicator().getImplicitContext().getContext()
58 ctx = dict(ctx)
59 ctx["omero.group"] = group
60 return ctx
61
63 if name.startswith("_"):
64 return self.__dict__[name]
65 elif hasattr(self._service, name):
66 method = getattr(self._service, name)
67 ctx = self._get_ctx(self._group_id)
68 return with_context(method, ctx)
69 raise AttributeError("'%s' object has no attribute '%s'" % (self.service, name))
70
71 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
72 """
73 Wrapper around a subprocess.Popen instance. Returned by ProcessorI
74 when a job is submitted. This implementation uses the given
75 interpreter to call a file that must be named "script" in the
76 generated temporary directory.
77
78 Call is equivalent to:
79
80 cd TMP_DIR
81 ICE_CONFIG=./config interpreter ./script >out 2>err &
82
83 The properties argument is used to generate the ./config file.
84
85 The params argument may be null in which case this process
86 is being used solely to calculate the parameters for the script
87 ("omero.scripts.parse=true")
88
89 If iskill is True, then on cleanup, this process will reap the
90 attached session completely.
91 """
92
93 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\
94 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast,\
95 omero_home = path.getcwd()):
96 """
97 Popen and callback_Cast are primarily for testing.
98 """
99 omero.util.SimpleServant.__init__(self, ctx)
100 self.omero_home = omero_home
101 self.interpreter = interpreter
102 self.properties = properties
103 self.params = params
104 self.iskill = iskill
105 self.Popen = Popen
106 self.callback_cast = callback_cast
107
108 self.rcode = None
109 self.callbacks = {}
110 self.popen = None
111 self.pid = None
112 self.started = None
113 self.stopped = None
114 self.final_status = None
115
116 self.uuid = properties["omero.user"]
117
118
119 self.make_files()
120 self.make_env()
121 self.make_config()
122 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
123
124
125
126
127
129 self.env = omero.util.Environment("PATH", "PYTHONPATH",\
130 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR", "HOME")
131
132
133
134
135
136
137
138 self.env.append("PYTHONPATH", str(self.omero_home / "lib" / "python"))
139 self.env.set("ICE_CONFIG", str(self.config_path))
140
142 self.dir = create_path("process", ".dir", folder = True)
143 self.script_path = self.dir / "script"
144 self.config_path = self.dir / "config"
145 self.stdout_path = self.dir / "out"
146 self.stderr_path = self.dir / "err"
147
149 """
150 Creates the ICE_CONFIG file used by the client.
151 """
152 config_file = open(str(self.config_path), "w")
153 try:
154 for key in self.properties.iterkeys():
155 config_file.write("%s=%s\n"%(key, self.properties[key]))
156 finally:
157 config_file.close()
158
160 """
161 Create a client for performing cleanup operations.
162 This client should be closed as soon as possible
163 by the process
164 """
165 try:
166 client = omero.client(["--Ice.Config=%s" % str(self.config_path)])
167 client.setAgent("OMERO.process")
168 client.createSession().detachOnDestroy()
169 self.logger.debug("client: %s" % client.sf)
170 return client
171 except:
172 self.logger.error("Failed to create client for %s" % self.uuid)
173 return None
174
175
176
177
178
179 @locked
181 """
182 Process creation has to wait until all external downloads, etc
183 are finished.
184 """
185
186 if self.isActive():
187 raise omero.ApiUsageException(None, None, "Already activated")
188
189 self.stdout = open(str(self.stdout_path), "w")
190 self.stderr = open(str(self.stderr_path), "w")
191 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr)
192 self.pid = self.popen.pid
193 self.started = time.time()
194 self.stopped = None
195 self.status("Activated")
196
197 @locked
199 """
200 Cleans up the temporary directory used by the process, and terminates
201 the Popen process if running.
202 """
203
204 if not self.isActive():
205 raise omero.ApiUsageException(None, None, "Not active")
206
207 if self.stopped:
208
209 return
210
211 self.stopped = time.time()
212 d_start = time.time()
213 self.status("Deactivating")
214
215
216 try:
217
218 self.shutdown()
219 self.popen = None
220
221 client = self.tmp_client()
222 try:
223 self.set_job_status(client)
224 self.cleanup_output()
225 self.upload_output(client)
226 self.cleanup_tmpdir()
227 finally:
228 if client:
229 client.__del__()
230
231 except exceptions.Exception:
232 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True)
233
234 d_stop = time.time()
235 elapsed = int(self.stopped - self.started)
236 d_elapsed = int(d_stop - d_start)
237 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
238
239 @locked
241 """
242 Tests only if this instance has a non-None popen attribute. After activation
243 this method will return True until the popen itself returns a non-None
244 value (self.rcode) at which time it will be nulled and this method will again
245 return False
246 """
247 return self.popen is not None
248
249 @locked
251 """
252 Returns true only if this instance has either a non-null
253 popen or a non-null rcode field.
254 """
255 return self.popen is not None or self.rcode is not None
256
257 @locked
259 return self.popen is not None and self.rcode is None
260
261 @locked
263 return self.rcode is not None
264
265 @locked
267 """
268 Allows short-cutting various checks if we already
269 have a rcode for this popen. A non-None return value
270 implies that a process was started and returned
271 the given non-None value itself.
272 """
273 if not self.wasActivated:
274 raise omero.InternalException(None, None, "Process never activated")
275 return self.isFinished()
276
277
278
279
280
283
284 @perf
285 @locked
287 """
288 Called periodically to keep the session alive. Returns
289 False if this resource can be cleaned up. (Resources API)
290 """
291
292 if not self.wasActivated():
293 return True
294
295 try:
296 self.poll()
297 self.ctx.getSession().getSessionService().getSession(self.uuid)
298 return True
299 except:
300 self.status("Keep alive failed")
301 return False
302
303 @perf
304 @locked
306 """
307 Deactivates the process (if active) and cleanups the server
308 connection. (Resources API)
309 """
310
311 if self.isRunning():
312 self.deactivate()
313
314 if not self.iskill:
315 return
316
317 try:
318 sf = self.ctx.getSession(recreate = False)
319 except:
320 self.logger.debug("Can't get session for cleanup")
321 return
322
323 self.status("Killing session")
324 svc = sf.getSessionService()
325 obj = omero.model.SessionI()
326 obj.uuid = omero.rtypes.rstring(self.uuid)
327 try:
328 while svc.closeSession(obj) > 0:
329 pass
330
331
332 except:
333 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
334
336 """
337 Flush and close the stderr and stdout streams.
338 """
339 try:
340 if hasattr(self, "stderr"):
341 self.stderr.flush()
342 self.stderr.close()
343 except:
344 self.logger.error("cleanup of sterr failed", exc_info = True)
345 try:
346 if hasattr(self, "stdout"):
347 self.stdout.flush()
348 self.stdout.close()
349 except:
350 self.logger.error("cleanup of sterr failed", exc_info = True)
351
371
373 """
374 If this is not a params calculation (i.e. parms != null) and the
375 stdout or stderr are non-null, they they will be uploaded and
376 attached to the job.
377 """
378 if not client:
379 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid)
380 return
381
382 if self.params:
383 out_format = self.params.stdoutFormat
384 err_format = self.params.stderrFormat
385 else:
386 out_format = "text/plain"
387 err_format = out_format
388
389 self._upload(client, self.stdout_path, "stdout", out_format)
390 self._upload(client, self.stderr_path, "stderr", err_format)
391
392 - def _upload(self, client, filename, name, format):
393
394 if not format:
395 return
396
397 filename = str(filename)
398 sz = os.path.getsize(filename)
399 if not sz:
400 self.status("No %s" % name)
401 return
402
403 try:
404 ofile = client.upload(filename, name=name, type=format)
405 jobid = long(client.getProperty("omero.job"))
406 link = omero.model.JobOriginalFileLinkI()
407 if self.params is None:
408 link.parent = omero.model.ParseJobI(rlong(jobid), False)
409 else:
410 link.parent = omero.model.ScriptJobI(rlong(jobid), False)
411 link.child = ofile
412 client.getSession().getUpdateService().saveObject(link)
413 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val))
414 except:
415 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
416
418 """
419 Remove all known files and finally the temporary directory.
420 If other files exist, an exception will be raised.
421 """
422 try:
423 remove_path(self.dir)
424 except:
425 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
426
427
428
429
430
435
436 @perf
437 @remoted
438 - def poll(self, current = None):
439 """
440 Checks popen.poll() (if active) and notifies all callbacks
441 if necessary. If this method returns a non-None value, then
442 the process will be marked inactive.
443 """
444
445 if self.alreadyDone():
446 return rint(self.rcode)
447
448 self.status("Polling")
449 if self.rcode is None:
450
451 return None
452 else:
453 self.deactivate()
454 rv = rint(self.rcode)
455 self.allcallbacks("processFinished", self.rcode)
456 return rv
457
458 @perf
459 @remoted
460 - def wait(self, current = None):
461 """
462 Waits on popen.wait() to return (if active) and notifies
463 all callbacks. Marks this process as inactive.
464 """
465
466 if self.alreadyDone():
467 return self.rcode
468
469 self.status("Waiting")
470 self.rcode = self.popen.wait()
471 self.deactivate()
472 self.allcallbacks("processFinished", self.rcode)
473 return self.rcode
474
476 """
477 Attempts to cancel the process by sending SIGTERM
478 (or similar)
479 """
480 try:
481 self.status("os.kill(TERM)")
482 os.kill(self.popen.pid, signal.SIGTERM)
483 except AttributeError:
484 self.logger.debug("No os.kill(TERM). Skipping cancel")
485
486 - def _send(self, iskill):
487 """
488 Helper method for sending signals. This method only
489 makes a call is the process is active.
490 """
491 if self.isRunning():
492 try:
493 if self.popen.poll() is None:
494 if iskill:
495 self.status("popen.kill(True)")
496 self.popen.kill(True)
497 else:
498 self._term()
499
500 else:
501 self.status("Skipped signal")
502 except OSError, oserr:
503 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
504
505 @perf
506 @remoted
507 - def cancel(self, current = None):
522
523 @perf
524 @remoted
525 - def kill(self, current = None):
537
538 @perf
539 @remoted
541 """
542 If self.popen is active, then first call cancel, wait a period of
543 time, and finally call kill.
544 """
545
546 if self.alreadyDone():
547 return
548
549 self.status("Shutdown")
550 try:
551 for i in range(5, 0, -1):
552 if self.cancel():
553 break
554 else:
555 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1)
556 self.stop_event.wait(6)
557 self.kill()
558 except:
559 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
560
561
562
563
564
565 @remoted
566 @locked
568 try:
569 id = callback.ice_getIdentity()
570 key = "%s/%s" % (id.category, id.name)
571 callback = callback.ice_oneway()
572 callback = self.callback_cast(callback)
573 if not callback:
574 e = "Callback is invalid"
575 else:
576 self.callbacks[key] = callback
577 self.logger.debug("Added callback: %s", key)
578 return
579 except exceptions.Exception, ex:
580 e = ex
581
582 msg = "Failed to add callback: %s. Reason: %s" % (callback, e)
583 self.logger.debug(msg)
584 raise omero.ApiUsageException(None, None, msg)
585
586 @remoted
587 @locked
589 try:
590 id = callback.ice_getIdentity()
591 key = "%s/%s" % (id.category, id.name)
592 if not key in self.callback:
593 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key)
594 del self.callbacks[key]
595 self.logger.debug("Removed callback: %s", key)
596 except exceptions.Exception, e:
597 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e)
598 self.logger.debug(msg)
599 raise omero.ApiUsageException(None, None, msg)
600
601 @locked
603 self.status("Callback %s" % method)
604 for key, cb in self.callbacks.items():
605 try:
606 m = getattr(cb, method)
607 m(arg)
608 except Ice.LocalException, e:
609 self.logger.debug("LocalException calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = False)
610 except:
611 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
612
614 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, (self.rcode is None and "-" or self.rcode), self.uuid)
615
617
620
622 try:
623 self.sf.keepAlive(None)
624 return True
625 except:
626 return False
627
630
631 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
632
633 - def __init__(self, ctx, needs_session = True,
634 use_session = None, accepts_list = None, cfg = None,
635 omero_home = path.getcwd()):
636
637 if accepts_list is None: accepts_list = []
638
639 self.omero_home = omero_home
640
641
642
643 self.use_session = use_session
644 """
645 If set, this session will be returned from internal_session and
646 the "needs_session" setting ignored.
647 """
648
649 if self.use_session:
650 needs_session = False
651
652 self.accepts_list = accepts_list
653 """
654 A list of contexts which will be accepted by this user-mode
655 processor.
656 """
657
658 omero.util.Servant.__init__(self, ctx, needs_session = needs_session)
659 if cfg is None:
660 self.cfg = os.path.join(omero_home, "etc", "ice.config")
661 self.cfg = os.path.abspath(self.cfg)
662 else:
663 self.cfg = cfg
664
665
666 self.resources.add( UseSessionHolder(use_session) )
667
682
684 """
685 Creates an omero.client instance for use by
686 users.
687 """
688 args = ["--Ice.Config=%s" % (self.cfg)]
689 rtr = self.internal_session().ice_getRouter()
690 if rtr:
691 args.insert(0, "--Ice.Default.Router=%s" % rtr)
692 client = omero.client(args)
693 client.setAgent(agent)
694 return client
695
697 """
698 Returns the session which should be used for lookups by this instance.
699 Some methods will create a session based on the session parameter.
700 In these cases, the session will belong to the user who is running a
701 script.
702 """
703 if self.use_session:
704 return self.use_session
705 else:
706 return self.ctx.getSession()
707
709 self.logger.info("Registering processor %s", self.prx)
710 prx = omero.grid.ProcessorPrx.uncheckedCast(self.prx)
711 session.sharedResources().addProcessor(prx)
712
731
732 @remoted
733 - def willAccept(self, userContext, groupContext, scriptContext, cb, current = None):
734
735 userID = None
736 if userContext != None:
737 userID = userContext.id.val
738
739 groupID = None
740 if groupContext != None:
741 groupID = groupContext.id.val
742
743 scriptID = None
744 if scriptContext != None:
745 scriptID = scriptContext.id.val
746
747 if scriptID:
748 try:
749 file, handle = self.lookup(scriptContext)
750 handle.close()
751 valid = (file is not None)
752 except:
753 self.logger.error("File lookup failed: user=%s, group=%s, script=%s",\
754 userID, groupID, scriptID, exc_info=1)
755 return
756 else:
757 valid = False
758 for x in self.accepts_list:
759 if isinstance(x, omero.model.Experimenter) and x.id.val == userID:
760 valid = True
761 elif isinstance(x, omero.model.ExperimenterGroup) and x.id.val == groupID:
762 valid = True
763
764 self.logger.debug("Accepts called on: user:%s group:%s scriptjob:%s - Valid: %s",
765 userID, groupID, scriptID, valid)
766
767 try:
768 id = self.internal_session().ice_getIdentity().name
769 cb = cb.ice_oneway()
770 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb)
771 cb.isAccepted(valid, id, str(self.prx))
772 except exceptions.Exception, e:
773 self.logger.warn("callback failed on willAccept: %s Exception:%s", cb, e)
774
775 return valid
776
777 @remoted
779
780 try:
781 cb = cb.ice_oneway()
782 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb)
783 servants = list(self.ctx.servant_map.values())
784 rv = []
785 for x in servants:
786 if hasattr(x, "properties"):
787 rv.append(long(x))
788 cb.responseRunning(rv)
789 except exceptions.Exception, e:
790 self.logger.warn("callback failed on requestRunning: %s Exception:%s", cb, e)
791
792
793 @remoted
794 - def parseJob(self, session, job, current = None):
814
815 @remoted
816 - def processJob(self, session, params, job, current = None):
828
829
830 @perf
831 - def process(self, client, session, job, current, params, properties = None, iskill = True):
832 """
833 session: session uuid, used primarily if client is None
834 client: an omero.client object which should be attached to a session
835 """
836
837 if properties is None: properties = {}
838
839 if not session or not job or not job.id:
840 raise omero.ApiUsageException("No null arguments")
841
842 file, handle = self.lookup(job)
843
844 try:
845 if not file:
846 raise omero.ApiUsageException(\
847 None, None, "Job should have one executable file attached.")
848
849 sf = self.internal_session()
850 if params:
851 self.logger.debug("Checking params for job %s" % job.id.val)
852 svc = sf.getSessionService()
853 inputs = svc.getInputs(session)
854 errors = omero.scripts.validate_inputs(params, inputs, svc, session)
855 if errors:
856 errors = "Invalid parameters:\n%s" % errors
857 raise omero.ValidationException(None, None, errors)
858
859 properties["omero.job"] = str(job.id.val)
860 properties["omero.user"] = session
861 properties["omero.pass"] = session
862 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router")
863
864 process = ProcessI(self.ctx, sys.executable, properties, params, iskill, omero_home = self.omero_home)
865 self.resources.add(process)
866
867
868 scriptText = sf.getScriptService().getScriptText(file.id.val)
869 process.script_path.write_bytes(scriptText)
870
871 self.logger.info("Downloaded file: %s" % file.id.val)
872 s = client.sha1(str(process.script_path))
873 if not s == file.sha1.val:
874 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s)
875 self.logger.error(msg)
876 process.cleanup()
877 raise omero.InternalException(None, None, msg)
878 else:
879 process.activate()
880 handle.setStatus("Running")
881
882 prx = self.ctx.add_servant(current, process)
883 return omero.grid.ProcessPrx.uncheckedCast(prx), process
884
885 finally:
886 handle.close()
887
888 -def usermode_processor(client, serverid = "UsermodeProcessor",\
889 cfg = None, accepts_list = None, stop_event = None,\
890 omero_home = path.getcwd()):
891 """
892 Creates an activates a usermode processor for the given client.
893 It is the responsibility of the client to call "cleanup()" on
894 the ProcessorI implementation which is returned.
895
896 cfg is the path to an --Ice.Config-valid file or files. If none
897 is given, the value of ICE_CONFIG will be taken from the environment
898 if available. Otherwise, all properties will be taken from the client
899 instance.
900
901 accepts_list is the list of IObject instances which will be passed to
902 omero.api.IScripts.validateScript. If none is given, only the current
903 Experimenter's own object will be passed.
904
905 stop_event is an threading.Event. One will be acquired from
906 omero.util.concurrency.get_event if none is provided.
907 """
908
909 if cfg is None:
910 cfg = os.environ.get("ICE_CONFIG")
911
912 if accepts_list is None:
913 uid = client.sf.getAdminService().getEventContext().userId
914 accepts_list = [omero.model.ExperimenterI(uid, False)]
915
916 if stop_event is None:
917 stop_event = omero.util.concurrency.get_event(name="UsermodeProcessor")
918
919 ctx = omero.util.ServerContext(serverid, client.ic, stop_event)
920 impl = omero.processor.ProcessorI(ctx,
921 use_session=client.sf, accepts_list=accepts_list, cfg=cfg,
922 omero_home = omero_home)
923 ctx.add_servant(client.adapter, impl)
924 return impl
925