Package omero :: Module processor
[hide private]
[frames] | no frames]

Source Code for Module omero.processor

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Grid Processor 
  4  # Copyright 2008 Glencoe Software, Inc.  All Rights Reserved. 
  5  # Use is subject to license terms supplied in LICENSE.txt 
  6  # 
  7   
  8  import os 
  9  import sys 
 10  import time 
 11  import signal 
 12  import logging 
 13  import traceback 
 14  import exceptions 
 15  import killableprocess as subprocess 
 16   
 17  from path import path 
 18   
 19  import Ice 
 20  import omero 
 21  import omero.clients 
 22  import omero.scripts 
 23  import omero.util 
 24  import omero.util.concurrency 
 25   
 26  from omero.util.temp_files import create_path, remove_path 
 27  from omero.util.decorators import remoted, perf, locked 
 28  from omero.rtypes import * 
 29  from omero.util.decorators import remoted, perf, wraps 
30 31 -def with_context(func, context):
32 """ Decorator for invoking Ice methods with a context """ 33 def handler(*args, **kwargs): 34 args = list(args) 35 args.append(context) 36 return func(*args, **kwargs)
37 handler = wraps(func)(handler) 38 return handler 39
40 -class WithGroup(object):
41 """ 42 Wraps a ServiceInterfacePrx instance and applies 43 a "omero.group" to the passed context on every 44 invotation. 45 46 For example, using a job handle as root requires logging 47 manually into the group. (ticket:2044) 48 """ 49
50 - def __init__(self, service, group_id):
51 self._service = service 52 self._group_id = str(group_id)
53
54 - def _get_ctx(self, group = None):
55 ctx = self._service.ice_getCommunicator().getImplicitContext().getContext() 56 ctx = dict(ctx) 57 ctx["omero.group"] = group 58 return ctx
59
60 - def __getattr__(self, name):
61 if name.startswith("_"): 62 return self.__dict__[name] 63 elif hasattr(self._service, name): 64 method = getattr(self._service, name) 65 ctx = self._get_ctx(self._group_id) 66 return with_context(method, ctx) 67 raise AttributeError("'%s' object has no attribute '%s'" % (self.service, name))
68
69 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
70 """ 71 Wrapper around a subprocess.Popen instance. Returned by ProcessorI 72 when a job is submitted. This implementation uses the given 73 interpreter to call a file that must be named "script" in the 74 generated temporary directory. 75 76 Call is equivalent to: 77 78 cd TMP_DIR 79 ICE_CONFIG=./config interpreter ./script >out 2>err & 80 81 The properties argument is used to generate the ./config file. 82 83 The params argument may be null in which case this process 84 is being used solely to calculate the parameters for the script 85 ("omero.scripts.parse=true") 86 87 If iskill is True, then on cleanup, this process will reap the 88 attached session completely. 89 """ 90
91 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\ 92 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast,\ 93 omero_home = path.getcwd()):
94 """ 95 Popen and callback_Cast are primarily for testing. 96 """ 97 omero.util.SimpleServant.__init__(self, ctx) 98 self.omero_home = omero_home #: Location for OMERO_HOME/lib/python 99 self.interpreter = interpreter #: Executable which will be used on the script 100 self.properties = properties #: Properties used to create an Ice.Config 101 self.params = params #: JobParams for this script. Possibly None if a ParseJob 102 self.iskill = iskill #: Whether or not, cleanup should kill the session 103 self.Popen = Popen #: Function which should be used for creating processes 104 self.callback_cast = callback_cast #: Function used to cast all ProcessCallback proxies 105 # Non arguments (mutable state) 106 self.rcode = None #: return code from popen 107 self.callbacks = {} #: dictionary from id strings to callback proxies 108 self.popen = None #: process. if None, then this instance isn't alive. 109 self.pid = None #: pid of the process. Once set, isn't nulled. 110 self.started = None #: time the process started 111 self.stopped = None #: time of deactivation 112 self.final_status = None #: status which will be sent on set_job_status 113 # Non arguments (immutable state) 114 self.uuid = properties["omero.user"] #: session this instance is tied to 115 116 # More fields set by these methods 117 self.make_files() 118 self.make_env() 119 self.make_config() 120 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
121 122 # 123 # Initialization methods 124 # 125
126 - def make_env(self):
127 self.env = omero.util.Environment("PATH", "PYTHONPATH",\ 128 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR", "HOME") 129 # WORKAROUND 130 # Currently duplicating the logic here as in the PYTHONPATH 131 # setting of the grid application descriptor (see etc/grid/*.xml) 132 # This should actually be taken care of in the descriptor itself 133 # by having setting PYTHONPATH to an absolute value. This is 134 # not currently possible with IceGrid (without using icepatch -- 135 # see 39.17.2 "node.datadir). 136 self.env.append("PYTHONPATH", str(self.omero_home / "lib" / "python")) 137 self.env.set("ICE_CONFIG", str(self.config_path))
138
139 - def make_files(self):
140 self.dir = create_path("process", ".dir", folder = True) 141 self.script_path = self.dir / "script" 142 self.config_path = self.dir / "config" 143 self.stdout_path = self.dir / "out" 144 self.stderr_path = self.dir / "err"
145
146 - def make_config(self):
147 """ 148 Creates the ICE_CONFIG file used by the client. 149 """ 150 config_file = open(str(self.config_path), "w") 151 try: 152 for key in self.properties.iterkeys(): 153 config_file.write("%s=%s\n"%(key, self.properties[key])) 154 finally: 155 config_file.close()
156
157 - def tmp_client(self):
158 """ 159 Create a client for performing cleanup operations. 160 This client should be closed as soon as possible 161 by the process 162 """ 163 try: 164 client = omero.client(["--Ice.Config=%s" % str(self.config_path)]) 165 client.setAgent("OMERO.process") 166 client.createSession().detachOnDestroy() 167 self.logger.debug("client: %s" % client.sf) 168 return client 169 except: 170 self.logger.error("Failed to create client for %s" % self.uuid) 171 return None
172 173 # 174 # Activation / Deactivation 175 # 176 177 @locked
178 - def activate(self):
179 """ 180 Process creation has to wait until all external downloads, etc 181 are finished. 182 """ 183 184 if self.isActive(): 185 raise omero.ApiUsageException(None, None, "Already activated") 186 187 self.stdout = open(str(self.stdout_path), "w") 188 self.stderr = open(str(self.stderr_path), "w") 189 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr) 190 self.pid = self.popen.pid 191 self.started = time.time() 192 self.stopped = None 193 self.status("Activated")
194 195 @locked
196 - def deactivate(self):
197 """ 198 Cleans up the temporary directory used by the process, and terminates 199 the Popen process if running. 200 """ 201 202 if not self.isActive(): 203 raise omero.ApiUsageException(None, None, "Not active") 204 205 if self.stopped: 206 # Prevent recursion since we are reusing kill & cancel 207 return 208 209 self.stopped = time.time() 210 d_start = time.time() 211 self.status("Deactivating") 212 213 # None of these should throw, but just in case 214 try: 215 216 self.shutdown() # Calls cancel & kill which recall this method! 217 self.popen = None # Now we are finished 218 219 client = self.tmp_client() 220 try: 221 self.set_job_status(client) 222 self.cleanup_output() 223 self.upload_output(client) # Important! 224 self.cleanup_tmpdir() 225 finally: 226 if client: 227 client.__del__() # Safe closeSession 228 229 except exceptions.Exception: 230 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True) 231 232 d_stop = time.time() 233 elapsed = int(self.stopped - self.started) 234 d_elapsed = int(d_stop - d_start) 235 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
236 237 @locked
238 - def isActive(self):
239 """ 240 Tests only if this instance has a non-None popen attribute. After activation 241 this method will return True until the popen itself returns a non-None 242 value (self.rcode) at which time it will be nulled and this method will again 243 return False 244 """ 245 return self.popen is not None
246 247 @locked
248 - def wasActivated(self):
249 """ 250 Returns true only if this instance has either a non-null 251 popen or a non-null rcode field. 252 """ 253 return self.popen is not None or self.rcode is not None
254 255 @locked
256 - def isRunning(self):
257 return self.popen is not None and self.rcode is None
258 259 @locked
260 - def isFinished(self):
261 return self.rcode is not None
262 263 @locked
264 - def alreadyDone(self):
265 """ 266 Allows short-cutting various checks if we already 267 have a rcode for this popen. A non-None return value 268 implies that a process was started and returned 269 the given non-None value itself. 270 """ 271 if not self.wasActivated: 272 raise omero.InternalException(None, None, "Process never activated") 273 return self.isFinished()
274 275 # 276 # Cleanup methods 277 # 278
279 - def __del__(self):
280 self.cleanup()
281 282 @perf 283 @locked
284 - def check(self):
285 """ 286 Called periodically to keep the session alive. Returns 287 False if this resource can be cleaned up. (Resources API) 288 """ 289 290 if not self.wasActivated(): 291 return True # This should only happen on startup, so ignore 292 293 try: 294 self.poll() 295 self.ctx.getSession().getSessionService().getSession(self.uuid) 296 return True 297 except: 298 self.status("Keep alive failed") 299 return False
300 301 @perf 302 @locked
303 - def cleanup(self):
304 """ 305 Deactivates the process (if active) and cleanups the server 306 connection. (Resources API) 307 """ 308 309 if self.isRunning(): 310 self.deactivate() 311 312 if not self.iskill: 313 return 314 315 try: 316 sf = self.ctx.getSession(recreate = False) 317 except: 318 self.logger.debug("Can't get session for cleanup") 319 return 320 321 self.status("Killing session") 322 svc = sf.getSessionService() 323 obj = omero.model.SessionI() 324 obj.uuid = omero.rtypes.rstring(self.uuid) 325 try: 326 while svc.closeSession(obj) > 0: 327 pass 328 # No action to be taken when iskill == False if 329 # we don't have an actual client to worry with. 330 except: 331 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
332
333 - def cleanup_output(self):
334 """ 335 Flush and close the stderr and stdout streams. 336 """ 337 try: 338 if hasattr(self, "stderr"): 339 self.stderr.flush() 340 self.stderr.close() 341 except: 342 self.logger.error("cleanup of sterr failed", exc_info = True) 343 try: 344 if hasattr(self, "stdout"): 345 self.stdout.flush() 346 self.stdout.close() 347 except: 348 self.logger.error("cleanup of sterr failed", exc_info = True)
349
350 - def set_job_status(self, client):
351 """ 352 Sets the job status 353 """ 354 if not client: 355 self.logger.error("No client: Cannot set job status for pid=%s (%s)", self.pid, self.uuid) 356 return 357 358 gid = client.sf.getAdminService().getEventContext().groupId 359 handle = WithGroup(client.sf.createJobHandle(), gid) 360 try: 361 status = self.final_status 362 if status is None: 363 status = ( self.rcode == 0 and "Finished" or "Error" ) 364 handle.attach(long(self.properties["omero.job"])) 365 oldStatus = handle.setStatus(status) 366 self.status("Changed job status from %s to %s" % (oldStatus, status)) 367 finally: 368 handle.close()
369
370 - def upload_output(self, client):
371 """ 372 If this is not a params calculation (i.e. parms != null) and the 373 stdout or stderr are non-null, they they will be uploaded and 374 attached to the job. 375 """ 376 if not client: 377 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid) 378 return 379 380 if self.params: 381 out_format = self.params.stdoutFormat 382 err_format = self.params.stderrFormat 383 else: 384 out_format = "text/plain" 385 err_format = out_format 386 387 self._upload(client, self.stdout_path, "stdout", out_format) 388 self._upload(client, self.stderr_path, "stderr", err_format)
389
390 - def _upload(self, client, filename, name, format):
391 392 if not format: 393 return 394 395 filename = str(filename) # Might be path.path 396 sz = os.path.getsize(filename) 397 if not sz: 398 self.status("No %s" % name) 399 return 400 401 try: 402 ofile = client.upload(filename, name=name, type=format) 403 jobid = long(client.getProperty("omero.job")) 404 link = omero.model.JobOriginalFileLinkI() 405 if self.params is None: 406 link.parent = omero.model.ParseJobI(rlong(jobid), False) 407 else: 408 link.parent = omero.model.ScriptJobI(rlong(jobid), False) 409 link.child = ofile 410 client.getSession().getUpdateService().saveObject(link) 411 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val)) 412 except: 413 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
414
415 - def cleanup_tmpdir(self):
416 """ 417 Remove all known files and finally the temporary directory. 418 If other files exist, an exception will be raised. 419 """ 420 try: 421 remove_path(self.dir) 422 except: 423 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
424 425 # 426 # popen methods 427 # 428
429 - def status(self, msg = ""):
430 if self.isRunning(): 431 self.rcode = self.popen.poll() 432 self.logger.info("%s : %s", self, msg)
433 434 @perf 435 @remoted
436 - def poll(self, current = None):
437 """ 438 Checks popen.poll() (if active) and notifies all callbacks 439 if necessary. If this method returns a non-None value, then 440 the process will be marked inactive. 441 """ 442 443 if self.alreadyDone(): 444 return rint(self.rcode) 445 446 self.status("Polling") 447 if self.rcode is None: 448 # Haven't finished yet, so do nothing. 449 return None 450 else: 451 self.deactivate() 452 rv = rint(self.rcode) 453 self.allcallbacks("processFinished", self.rcode) 454 return rv
455 456 @perf 457 @remoted
458 - def wait(self, current = None):
459 """ 460 Waits on popen.wait() to return (if active) and notifies 461 all callbacks. Marks this process as inactive. 462 """ 463 464 if self.alreadyDone(): 465 return self.rcode 466 467 self.status("Waiting") 468 self.rcode = self.popen.wait() 469 self.deactivate() 470 self.allcallbacks("processFinished", self.rcode) 471 return self.rcode
472
473 - def _term(self):
474 """ 475 Attempts to cancel the process by sending SIGTERM 476 (or similar) 477 """ 478 try: 479 self.status("os.kill(TERM)") 480 os.kill(self.popen.pid, signal.SIGTERM) 481 except AttributeError: 482 self.logger.debug("No os.kill(TERM). Skipping cancel")
483
484 - def _send(self, iskill):
485 """ 486 Helper method for sending signals. This method only 487 makes a call is the process is active. 488 """ 489 if self.isRunning(): 490 try: 491 if self.popen.poll() is None: 492 if iskill: 493 self.status("popen.kill(True)") 494 self.popen.kill(True) 495 else: 496 self._term() 497 498 else: 499 self.status("Skipped signal") 500 except OSError, oserr: 501 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
502 503 @perf 504 @remoted
505 - def cancel(self, current = None):
506 """ 507 Tries to cancel popen (if active) and notifies callbacks. 508 """ 509 510 if self.alreadyDone(): 511 return True 512 513 self.final_status = "Cancelled" 514 self._send(iskill=False) 515 finished = self.isFinished() 516 if finished: 517 self.deactivate() 518 self.allcallbacks("processCancelled", finished) 519 return finished
520 521 @perf 522 @remoted
523 - def kill(self, current = None):
524 525 if self.alreadyDone(): 526 return True 527 528 self.final_status = "Cancelled" 529 self._send(iskill=True) 530 finished = self.isFinished() 531 if finished: 532 self.deactivate() 533 self.allcallbacks("processKilled", finished) 534 return finished
535 536 @perf 537 @remoted
538 - def shutdown(self, current = None):
539 """ 540 If self.popen is active, then first call cancel, wait a period of 541 time, and finally call kill. 542 """ 543 544 if self.alreadyDone(): 545 return 546 547 self.status("Shutdown") 548 try: 549 for i in range(5, 0, -1): 550 if self.cancel(): 551 break 552 else: 553 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1) 554 self.stop_event.wait(6) 555 self.kill() 556 except: 557 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
558 559 # 560 # Callbacks 561 # 562 563 @remoted 564 @locked
565 - def registerCallback(self, callback, current = None):
566 try: 567 id = callback.ice_getIdentity() 568 key = "%s/%s" % (id.category, id.name) 569 callback = callback.ice_oneway() 570 callback = self.callback_cast(callback) 571 if not callback: 572 e = "Callback is invalid" 573 else: 574 self.callbacks[key] = callback 575 self.logger.debug("Added callback: %s", key) 576 return 577 except exceptions.Exception, ex: 578 e = ex 579 # Only reached on failure 580 msg = "Failed to add callback: %s. Reason: %s" % (callback, e) 581 self.logger.debug(msg) 582 raise omero.ApiUsageException(None, None, msg)
583 584 @remoted 585 @locked
586 - def unregisterCallback(self, callback, current = None):
587 try: 588 id = callback.ice_getIdentity() 589 key = "%s/%s" % (id.category, id.name) 590 if not key in self.callback: 591 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key) 592 del self.callbacks[key] 593 self.logger.debug("Removed callback: %s", key) 594 except exceptions.Exception, e: 595 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e) 596 self.logger.debug(msg) 597 raise omero.ApiUsageException(None, None, msg)
598 599 @locked
600 - def allcallbacks(self, method, arg):
601 self.status("Callback %s" % method) 602 for key, cb in self.callbacks.items(): 603 try: 604 m = getattr(cb, method) 605 m(arg) 606 except Ice.LocalException, e: 607 self.logger.debug("LocalException calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = False) 608 except: 609 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
610
611 - def __str__(self):
612 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, (self.rcode is None and "-" or self.rcode), self.uuid)
613
614 -class UseSessionHolder(object):
615
616 - def __init__(self, sf):
617 self.sf = sf
618
619 - def check(self):
620 try: 621 self.sf.keepAlive(None) 622 return True 623 except: 624 return False
625
626 - def cleanup(self):
627 pass
628
629 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
630
631 - def __init__(self, ctx, needs_session = True, 632 use_session = None, accepts_list = [], cfg = None, 633 omero_home = path.getcwd()):
634 635 self.omero_home = omero_home 636 637 # Extensions for user-mode processors (ticket:1672) 638 639 self.use_session = use_session 640 """ 641 If set, this session will be returned from internal_session and 642 the "needs_session" setting ignored. 643 """ 644 645 if self.use_session: 646 needs_session = False 647 648 self.accepts_list = accepts_list 649 """ 650 A list of contexts which will be accepted by this user-mode 651 processor. 652 """ 653 654 omero.util.Servant.__init__(self, ctx, needs_session = needs_session) 655 if cfg is None: 656 self.cfg = os.path.join(os.curdir, "etc", "ice.config") 657 self.cfg = os.path.abspath(self.cfg) 658 else: 659 self.cfg = cfg 660 661 # Keep this session alive until the processor is finished 662 self.resources.add( UseSessionHolder(use_session) )
663
664 - def setProxy(self, prx):
665 """ 666 Overrides the default action in order to register this proxy 667 with the session's sharedResources to register for callbacks. 668 The on_newsession handler will also keep new sessions informed. 669 670 See ticket:2304 671 """ 672 omero.util.Servant.setProxy(self, prx) 673 session = self.internal_session() 674 self.register_session(session) 675 676 # Keep other session informed 677 self.ctx.on_newsession = self.register_session
678
679 - def user_client(self, agent):
680 """ 681 Creates an omero.client instance for use by 682 users. 683 """ 684 args = ["--Ice.Config=%s" % (self.cfg)] 685 rtr = self.internal_session().ice_getRouter() 686 if rtr: 687 args.insert(0, "--Ice.Default.Router=%s" % rtr) # FIXME : How do we find an internal router? 688 client = omero.client(args) 689 client.setAgent(agent) 690 return client
691
692 - def internal_session(self):
693 """ 694 Returns the session which should be used for lookups by this instance. 695 Some methods will create a session based on the session parameter. 696 In these cases, the session will belong to the user who is running a 697 script. 698 """ 699 if self.use_session: 700 return self.use_session 701 else: 702 return self.ctx.getSession()
703
704 - def register_session(self, session):
705 self.logger.info("Registering processor %s", self.prx) 706 prx = omero.grid.ProcessorPrx.uncheckedCast(self.prx) 707 session.sharedResources().addProcessor(prx)
708
709 - def lookup(self, job):
710 sf = self.internal_session() 711 gid = job.details.group.id.val 712 handle = WithGroup(sf.createJobHandle(), gid) 713 try: 714 handle.attach(job.id.val) 715 if handle.jobFinished(): 716 handle.close() 717 raise omero.ApiUsageException("Job already finished.") 718 719 prx = WithGroup(sf.getScriptService(), gid) 720 file = prx.validateScript(job, self.accepts_list) 721 722 except omero.SecurityViolation, sv: 723 self.logger.debug("SecurityViolation on validate job %s from group %s", job.id.val, gid) 724 file = None 725 726 return file, handle
727 728 @remoted
729 - def willAccept(self, userContext, groupContext, scriptContext, cb, current = None):
730 731 userID = None 732 if userContext != None: 733 userID = userContext.id.val 734 735 groupID = None 736 if groupContext != None: 737 groupID = groupContext.id.val 738 739 scriptID = None 740 if scriptContext != None: 741 scriptID = scriptContext.id.val 742 743 if scriptID: 744 try: 745 file, handle = self.lookup(scriptContext) 746 handle.close() 747 valid = (file is not None) 748 except: 749 self.logger.error("File lookup failed: user=%s, group=%s, script=%s",\ 750 userID, groupID, scriptID, exc_info=1) 751 return # EARlY EXIT ! 752 else: 753 valid = False 754 for x in self.accepts_list: 755 if isinstance(x, omero.model.Experimenter) and x.id.val == userID: 756 valid = True 757 elif isinstance(x, omero.model.ExperimenterGroup) and x.id.val == groupID: 758 valid = True 759 760 self.logger.debug("Accepts called on: user:%s group:%s scriptjob:%s - Valid: %s", 761 userID, groupID, scriptID, valid) 762 763 try: 764 id = self.internal_session().ice_getIdentity().name 765 cb = cb.ice_oneway() 766 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb) 767 cb.isAccepted(valid, id, str(self.prx)) 768 except exceptions.Exception, e: 769 self.logger.warn("callback failed on willAccept: %s Exception:%s", cb, e) 770 771 return valid
772 773 @remoted
774 - def requestRunning(self, cb, current = None):
775 776 try: 777 cb = cb.ice_oneway() 778 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb) 779 servants = list(self.ctx.servant_map.values()) 780 rv = [] 781 for x in servants: 782 if hasattr(x, "properties"): 783 rv.append(long(x)) 784 cb.responseRunning(rv) 785 except exceptions.Exception, e: 786 self.logger.warn("callback failed on requestRunning: %s Exception:%s", cb, e)
787 788 789 @remoted
790 - def parseJob(self, session, job, current = None):
791 self.logger.info("parseJob: Session = %s, JobId = %s" % (session, job.id.val)) 792 client = self.user_client("OMERO.parseJob") 793 794 try: 795 iskill = False 796 client.joinSession(session).detachOnDestroy() 797 properties = {} 798 properties["omero.scripts.parse"] = "true" 799 prx, process = self.process(client, session, job, current, None, properties, iskill) 800 process.wait() 801 rv = client.getOutput("omero.scripts.parse") 802 if rv != None: 803 return rv.val 804 else: 805 self.logger.warning("No output found for omero.scripts.parse. Keys: %s" % client.getOutputKeys()) 806 return None 807 finally: 808 client.closeSession() 809 del client
810 811 @remoted
812 - def processJob(self, session, params, job, current = None):
813 """ 814 """ 815 self.logger.info("processJob: Session = %s, JobId = %s" % (session, job.id.val)) 816 client = self.user_client("OMERO.processJob") 817 try: 818 client.joinSession(session).detachOnDestroy() 819 prx, process = self.process(client, session, job, current, params, iskill = True) 820 return prx 821 finally: 822 client.closeSession() 823 del client
824 825 826 @perf
827 - def process(self, client, session, job, current, params, properties = {}, iskill = True):
828 """ 829 session: session uuid, used primarily if client is None 830 client: an omero.client object which should be attached to a session 831 """ 832 833 if not session or not job or not job.id: 834 raise omero.ApiUsageException("No null arguments") 835 836 file, handle = self.lookup(job) 837 838 try: 839 if not file: 840 raise omero.ApiUsageException(\ 841 None, None, "Job should have one executable file attached.") 842 843 sf = self.internal_session() 844 if params: 845 self.logger.debug("Checking params for job %s" % job.id.val) 846 svc = sf.getSessionService() 847 inputs = svc.getInputs(session) 848 errors = omero.scripts.validate_inputs(params, inputs, svc, session) 849 if errors: 850 errors = "Invalid parameters:\n%s" % errors 851 raise omero.ValidationException(None, None, errors) 852 853 properties["omero.job"] = str(job.id.val) 854 properties["omero.user"] = session 855 properties["omero.pass"] = session 856 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router") 857 858 process = ProcessI(self.ctx, "python", properties, params, iskill, omero_home = self.omero_home) 859 self.resources.add(process) 860 861 # client.download(file, str(process.script_path)) 862 scriptText = sf.getScriptService().getScriptText(file.id.val) 863 process.script_path.write_bytes(scriptText) 864 865 self.logger.info("Downloaded file: %s" % file.id.val) 866 s = client.sha1(str(process.script_path)) 867 if not s == file.sha1.val: 868 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s) 869 self.logger.error(msg) 870 process.cleanup() 871 raise omero.InternalException(None, None, msg) 872 else: 873 process.activate() 874 handle.setStatus("Running") 875 876 prx = self.ctx.add_servant(current, process) 877 return omero.grid.ProcessPrx.uncheckedCast(prx), process 878 879 finally: 880 handle.close()
881
882 -def usermode_processor(client, serverid = "UsermodeProcessor",\ 883 cfg = None, accepts_list = None, stop_event = None,\ 884 omero_home = path.getcwd()):
885 """ 886 Creates an activates a usermode processor for the given client. 887 It is the responsibility of the client to call "cleanup()" on 888 the ProcessorI implementation which is returned. 889 890 cfg is the path to an --Ice.Config-valid file or files. If none 891 is given, the value of ICE_CONFIG will be taken from the environment 892 if available. Otherwise, all properties will be taken from the client 893 instance. 894 895 accepts_list is the list of IObject instances which will be passed to 896 omero.api.IScripts.validateScript. If none is given, only the current 897 Experimenter's own object will be passed. 898 899 stop_event is an threading.Event. One will be acquired from 900 omero.util.concurrency.get_event if none is provided. 901 """ 902 903 if cfg is None: 904 cfg = os.environ.get("ICE_CONFIG") 905 906 if accepts_list is None: 907 uid = client.sf.getAdminService().getEventContext().userId 908 accepts_list = [omero.model.ExperimenterI(uid, False)] 909 910 if stop_event is None: 911 stop_event = omero.util.concurrency.get_event(name="UsermodeProcessor") 912 913 ctx = omero.util.ServerContext(serverid, client.ic, stop_event) 914 impl = omero.processor.ProcessorI(ctx, 915 use_session=client.sf, accepts_list=accepts_list, cfg=cfg, 916 omero_home = omero_home) 917 ctx.add_servant(client.adapter, impl) 918 return impl
919