Package omero :: Module processor
[hide private]
[frames] | no frames]

Source Code for Module omero.processor

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Grid Processor 
  4  # Copyright 2008 Glencoe Software, Inc.  All Rights Reserved. 
  5  # Use is subject to license terms supplied in LICENSE.txt 
  6  # 
  7   
  8  import os 
  9  import time 
 10  import signal 
 11  import logging 
 12  import traceback 
 13  import exceptions 
 14  import killableprocess as subprocess 
 15   
 16  from path import path 
 17   
 18  import Ice 
 19  import omero 
 20  import omero.clients 
 21  import omero.scripts 
 22  import omero.util 
 23  import omero.util.concurrency 
 24   
 25  from omero.util.temp_files import create_path, remove_path 
 26  from omero.util.decorators import remoted, perf, locked 
 27  from omero.rtypes import * 
 28  from omero.util.decorators import remoted, perf, wraps 
 29   
 30  sys = __import__("sys") 
31 32 33 -def with_context(func, context):
34 """ Decorator for invoking Ice methods with a context """ 35 def handler(*args, **kwargs): 36 args = list(args) 37 args.append(context) 38 return func(*args, **kwargs)
39 handler = wraps(func)(handler) 40 return handler 41
42 -class WithGroup(object):
43 """ 44 Wraps a ServiceInterfacePrx instance and applies 45 a "omero.group" to the passed context on every 46 invotation. 47 48 For example, using a job handle as root requires logging 49 manually into the group. (ticket:2044) 50 """ 51
52 - def __init__(self, service, group_id):
53 self._service = service 54 self._group_id = str(group_id)
55
56 - def _get_ctx(self, group = None):
57 ctx = self._service.ice_getCommunicator().getImplicitContext().getContext() 58 ctx = dict(ctx) 59 ctx["omero.group"] = group 60 return ctx
61
62 - def __getattr__(self, name):
63 if name.startswith("_"): 64 return self.__dict__[name] 65 elif hasattr(self._service, name): 66 method = getattr(self._service, name) 67 ctx = self._get_ctx(self._group_id) 68 return with_context(method, ctx) 69 raise AttributeError("'%s' object has no attribute '%s'" % (self.service, name))
70
71 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
72 """ 73 Wrapper around a subprocess.Popen instance. Returned by ProcessorI 74 when a job is submitted. This implementation uses the given 75 interpreter to call a file that must be named "script" in the 76 generated temporary directory. 77 78 Call is equivalent to: 79 80 cd TMP_DIR 81 ICE_CONFIG=./config interpreter ./script >out 2>err & 82 83 The properties argument is used to generate the ./config file. 84 85 The params argument may be null in which case this process 86 is being used solely to calculate the parameters for the script 87 ("omero.scripts.parse=true") 88 89 If iskill is True, then on cleanup, this process will reap the 90 attached session completely. 91 """ 92
93 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\ 94 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast,\ 95 omero_home = path.getcwd()):
96 """ 97 Popen and callback_Cast are primarily for testing. 98 """ 99 omero.util.SimpleServant.__init__(self, ctx) 100 self.omero_home = omero_home #: Location for OMERO_HOME/lib/python 101 self.interpreter = interpreter #: Executable which will be used on the script 102 self.properties = properties #: Properties used to create an Ice.Config 103 self.params = params #: JobParams for this script. Possibly None if a ParseJob 104 self.iskill = iskill #: Whether or not, cleanup should kill the session 105 self.Popen = Popen #: Function which should be used for creating processes 106 self.callback_cast = callback_cast #: Function used to cast all ProcessCallback proxies 107 # Non arguments (mutable state) 108 self.rcode = None #: return code from popen 109 self.callbacks = {} #: dictionary from id strings to callback proxies 110 self.popen = None #: process. if None, then this instance isn't alive. 111 self.pid = None #: pid of the process. Once set, isn't nulled. 112 self.started = None #: time the process started 113 self.stopped = None #: time of deactivation 114 self.final_status = None #: status which will be sent on set_job_status 115 # Non arguments (immutable state) 116 self.uuid = properties["omero.user"] #: session this instance is tied to 117 118 # More fields set by these methods 119 self.make_files() 120 self.make_env() 121 self.make_config() 122 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
123 124 # 125 # Initialization methods 126 # 127
128 - def make_env(self):
129 self.env = omero.util.Environment("PATH", "PYTHONPATH",\ 130 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR", "HOME") 131 # WORKAROUND 132 # Currently duplicating the logic here as in the PYTHONPATH 133 # setting of the grid application descriptor (see etc/grid/*.xml) 134 # This should actually be taken care of in the descriptor itself 135 # by having setting PYTHONPATH to an absolute value. This is 136 # not currently possible with IceGrid (without using icepatch -- 137 # see 39.17.2 "node.datadir). 138 self.env.append("PYTHONPATH", str(self.omero_home / "lib" / "python")) 139 self.env.set("ICE_CONFIG", str(self.config_path))
140
141 - def make_files(self):
142 self.dir = create_path("process", ".dir", folder = True) 143 self.script_path = self.dir / "script" 144 self.config_path = self.dir / "config" 145 self.stdout_path = self.dir / "out" 146 self.stderr_path = self.dir / "err"
147
148 - def make_config(self):
149 """ 150 Creates the ICE_CONFIG file used by the client. 151 """ 152 config_file = open(str(self.config_path), "w") 153 try: 154 for key in self.properties.iterkeys(): 155 config_file.write("%s=%s\n"%(key, self.properties[key])) 156 finally: 157 config_file.close()
158
159 - def tmp_client(self):
160 """ 161 Create a client for performing cleanup operations. 162 This client should be closed as soon as possible 163 by the process 164 """ 165 try: 166 client = omero.client(["--Ice.Config=%s" % str(self.config_path)]) 167 client.setAgent("OMERO.process") 168 client.createSession().detachOnDestroy() 169 self.logger.debug("client: %s" % client.sf) 170 return client 171 except: 172 self.logger.error("Failed to create client for %s" % self.uuid) 173 return None
174 175 # 176 # Activation / Deactivation 177 # 178 179 @locked
180 - def activate(self):
181 """ 182 Process creation has to wait until all external downloads, etc 183 are finished. 184 """ 185 186 if self.isActive(): 187 raise omero.ApiUsageException(None, None, "Already activated") 188 189 self.stdout = open(str(self.stdout_path), "w") 190 self.stderr = open(str(self.stderr_path), "w") 191 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr) 192 self.pid = self.popen.pid 193 self.started = time.time() 194 self.stopped = None 195 self.status("Activated")
196 197 @locked
198 - def deactivate(self):
199 """ 200 Cleans up the temporary directory used by the process, and terminates 201 the Popen process if running. 202 """ 203 204 if not self.isActive(): 205 raise omero.ApiUsageException(None, None, "Not active") 206 207 if self.stopped: 208 # Prevent recursion since we are reusing kill & cancel 209 return 210 211 self.stopped = time.time() 212 d_start = time.time() 213 self.status("Deactivating") 214 215 # None of these should throw, but just in case 216 try: 217 218 self.shutdown() # Calls cancel & kill which recall this method! 219 self.popen = None # Now we are finished 220 221 client = self.tmp_client() 222 try: 223 self.set_job_status(client) 224 self.cleanup_output() 225 self.upload_output(client) # Important! 226 self.cleanup_tmpdir() 227 finally: 228 if client: 229 client.__del__() # Safe closeSession 230 231 except exceptions.Exception: 232 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True) 233 234 d_stop = time.time() 235 elapsed = int(self.stopped - self.started) 236 d_elapsed = int(d_stop - d_start) 237 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
238 239 @locked
240 - def isActive(self):
241 """ 242 Tests only if this instance has a non-None popen attribute. After activation 243 this method will return True until the popen itself returns a non-None 244 value (self.rcode) at which time it will be nulled and this method will again 245 return False 246 """ 247 return self.popen is not None
248 249 @locked
250 - def wasActivated(self):
251 """ 252 Returns true only if this instance has either a non-null 253 popen or a non-null rcode field. 254 """ 255 return self.popen is not None or self.rcode is not None
256 257 @locked
258 - def isRunning(self):
259 return self.popen is not None and self.rcode is None
260 261 @locked
262 - def isFinished(self):
263 return self.rcode is not None
264 265 @locked
266 - def alreadyDone(self):
267 """ 268 Allows short-cutting various checks if we already 269 have a rcode for this popen. A non-None return value 270 implies that a process was started and returned 271 the given non-None value itself. 272 """ 273 if not self.wasActivated: 274 raise omero.InternalException(None, None, "Process never activated") 275 return self.isFinished()
276 277 # 278 # Cleanup methods 279 # 280
281 - def __del__(self):
282 self.cleanup()
283 284 @perf 285 @locked
286 - def check(self):
287 """ 288 Called periodically to keep the session alive. Returns 289 False if this resource can be cleaned up. (Resources API) 290 """ 291 292 if not self.wasActivated(): 293 return True # This should only happen on startup, so ignore 294 295 try: 296 self.poll() 297 self.ctx.getSession().getSessionService().getSession(self.uuid) 298 return True 299 except: 300 self.status("Keep alive failed") 301 return False
302 303 @perf 304 @locked
305 - def cleanup(self):
306 """ 307 Deactivates the process (if active) and cleanups the server 308 connection. (Resources API) 309 """ 310 311 if self.isRunning(): 312 self.deactivate() 313 314 if not self.iskill: 315 return 316 317 try: 318 sf = self.ctx.getSession(recreate = False) 319 except: 320 self.logger.debug("Can't get session for cleanup") 321 return 322 323 self.status("Killing session") 324 svc = sf.getSessionService() 325 obj = omero.model.SessionI() 326 obj.uuid = omero.rtypes.rstring(self.uuid) 327 try: 328 while svc.closeSession(obj) > 0: 329 pass 330 # No action to be taken when iskill == False if 331 # we don't have an actual client to worry with. 332 except: 333 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
334
335 - def cleanup_output(self):
336 """ 337 Flush and close the stderr and stdout streams. 338 """ 339 try: 340 if hasattr(self, "stderr"): 341 self.stderr.flush() 342 self.stderr.close() 343 except: 344 self.logger.error("cleanup of sterr failed", exc_info = True) 345 try: 346 if hasattr(self, "stdout"): 347 self.stdout.flush() 348 self.stdout.close() 349 except: 350 self.logger.error("cleanup of sterr failed", exc_info = True)
351
352 - def set_job_status(self, client):
353 """ 354 Sets the job status 355 """ 356 if not client: 357 self.logger.error("No client: Cannot set job status for pid=%s (%s)", self.pid, self.uuid) 358 return 359 360 gid = client.sf.getAdminService().getEventContext().groupId 361 handle = WithGroup(client.sf.createJobHandle(), gid) 362 try: 363 status = self.final_status 364 if status is None: 365 status = ( self.rcode == 0 and "Finished" or "Error" ) 366 handle.attach(long(self.properties["omero.job"])) 367 oldStatus = handle.setStatus(status) 368 self.status("Changed job status from %s to %s" % (oldStatus, status)) 369 finally: 370 handle.close()
371
372 - def upload_output(self, client):
373 """ 374 If this is not a params calculation (i.e. parms != null) and the 375 stdout or stderr are non-null, they they will be uploaded and 376 attached to the job. 377 """ 378 if not client: 379 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid) 380 return 381 382 if self.params: 383 out_format = self.params.stdoutFormat 384 err_format = self.params.stderrFormat 385 else: 386 out_format = "text/plain" 387 err_format = out_format 388 389 self._upload(client, self.stdout_path, "stdout", out_format) 390 self._upload(client, self.stderr_path, "stderr", err_format)
391
392 - def _upload(self, client, filename, name, format):
393 394 if not format: 395 return 396 397 filename = str(filename) # Might be path.path 398 sz = os.path.getsize(filename) 399 if not sz: 400 self.status("No %s" % name) 401 return 402 403 try: 404 ofile = client.upload(filename, name=name, type=format) 405 jobid = long(client.getProperty("omero.job")) 406 link = omero.model.JobOriginalFileLinkI() 407 if self.params is None: 408 link.parent = omero.model.ParseJobI(rlong(jobid), False) 409 else: 410 link.parent = omero.model.ScriptJobI(rlong(jobid), False) 411 link.child = ofile 412 client.getSession().getUpdateService().saveObject(link) 413 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val)) 414 except: 415 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
416
417 - def cleanup_tmpdir(self):
418 """ 419 Remove all known files and finally the temporary directory. 420 If other files exist, an exception will be raised. 421 """ 422 try: 423 remove_path(self.dir) 424 except: 425 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
426 427 # 428 # popen methods 429 # 430
431 - def status(self, msg = ""):
432 if self.isRunning(): 433 self.rcode = self.popen.poll() 434 self.logger.info("%s : %s", self, msg)
435 436 @perf 437 @remoted
438 - def poll(self, current = None):
439 """ 440 Checks popen.poll() (if active) and notifies all callbacks 441 if necessary. If this method returns a non-None value, then 442 the process will be marked inactive. 443 """ 444 445 if self.alreadyDone(): 446 return rint(self.rcode) 447 448 self.status("Polling") 449 if self.rcode is None: 450 # Haven't finished yet, so do nothing. 451 return None 452 else: 453 self.deactivate() 454 rv = rint(self.rcode) 455 self.allcallbacks("processFinished", self.rcode) 456 return rv
457 458 @perf 459 @remoted
460 - def wait(self, current = None):
461 """ 462 Waits on popen.wait() to return (if active) and notifies 463 all callbacks. Marks this process as inactive. 464 """ 465 466 if self.alreadyDone(): 467 return self.rcode 468 469 self.status("Waiting") 470 self.rcode = self.popen.wait() 471 self.deactivate() 472 self.allcallbacks("processFinished", self.rcode) 473 return self.rcode
474
475 - def _term(self):
476 """ 477 Attempts to cancel the process by sending SIGTERM 478 (or similar) 479 """ 480 try: 481 self.status("os.kill(TERM)") 482 os.kill(self.popen.pid, signal.SIGTERM) 483 except AttributeError: 484 self.logger.debug("No os.kill(TERM). Skipping cancel")
485
486 - def _send(self, iskill):
487 """ 488 Helper method for sending signals. This method only 489 makes a call is the process is active. 490 """ 491 if self.isRunning(): 492 try: 493 if self.popen.poll() is None: 494 if iskill: 495 self.status("popen.kill(True)") 496 self.popen.kill(True) 497 else: 498 self._term() 499 500 else: 501 self.status("Skipped signal") 502 except OSError, oserr: 503 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
504 505 @perf 506 @remoted
507 - def cancel(self, current = None):
508 """ 509 Tries to cancel popen (if active) and notifies callbacks. 510 """ 511 512 if self.alreadyDone(): 513 return True 514 515 self.final_status = "Cancelled" 516 self._send(iskill=False) 517 finished = self.isFinished() 518 if finished: 519 self.deactivate() 520 self.allcallbacks("processCancelled", finished) 521 return finished
522 523 @perf 524 @remoted
525 - def kill(self, current = None):
526 527 if self.alreadyDone(): 528 return True 529 530 self.final_status = "Cancelled" 531 self._send(iskill=True) 532 finished = self.isFinished() 533 if finished: 534 self.deactivate() 535 self.allcallbacks("processKilled", finished) 536 return finished
537 538 @perf 539 @remoted
540 - def shutdown(self, current = None):
541 """ 542 If self.popen is active, then first call cancel, wait a period of 543 time, and finally call kill. 544 """ 545 546 if self.alreadyDone(): 547 return 548 549 self.status("Shutdown") 550 try: 551 for i in range(5, 0, -1): 552 if self.cancel(): 553 break 554 else: 555 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1) 556 self.stop_event.wait(6) 557 self.kill() 558 except: 559 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
560 561 # 562 # Callbacks 563 # 564 565 @remoted 566 @locked
567 - def registerCallback(self, callback, current = None):
568 try: 569 id = callback.ice_getIdentity() 570 key = "%s/%s" % (id.category, id.name) 571 callback = callback.ice_oneway() 572 callback = self.callback_cast(callback) 573 if not callback: 574 e = "Callback is invalid" 575 else: 576 self.callbacks[key] = callback 577 self.logger.debug("Added callback: %s", key) 578 return 579 except exceptions.Exception, ex: 580 e = ex 581 # Only reached on failure 582 msg = "Failed to add callback: %s. Reason: %s" % (callback, e) 583 self.logger.debug(msg) 584 raise omero.ApiUsageException(None, None, msg)
585 586 @remoted 587 @locked
588 - def unregisterCallback(self, callback, current = None):
589 try: 590 id = callback.ice_getIdentity() 591 key = "%s/%s" % (id.category, id.name) 592 if not key in self.callback: 593 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key) 594 del self.callbacks[key] 595 self.logger.debug("Removed callback: %s", key) 596 except exceptions.Exception, e: 597 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e) 598 self.logger.debug(msg) 599 raise omero.ApiUsageException(None, None, msg)
600 601 @locked
602 - def allcallbacks(self, method, arg):
603 self.status("Callback %s" % method) 604 for key, cb in self.callbacks.items(): 605 try: 606 m = getattr(cb, method) 607 m(arg) 608 except Ice.LocalException, e: 609 self.logger.debug("LocalException calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = False) 610 except: 611 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
612
613 - def __str__(self):
614 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, (self.rcode is None and "-" or self.rcode), self.uuid)
615
616 -class UseSessionHolder(object):
617
618 - def __init__(self, sf):
619 self.sf = sf
620
621 - def check(self):
622 try: 623 self.sf.keepAlive(None) 624 return True 625 except: 626 return False
627
628 - def cleanup(self):
629 pass
630
631 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
632
633 - def __init__(self, ctx, needs_session = True, 634 use_session = None, accepts_list = None, cfg = None, 635 omero_home = path.getcwd()):
636 637 if accepts_list is None: accepts_list = [] 638 639 self.omero_home = omero_home 640 641 # Extensions for user-mode processors (ticket:1672) 642 643 self.use_session = use_session 644 """ 645 If set, this session will be returned from internal_session and 646 the "needs_session" setting ignored. 647 """ 648 649 if self.use_session: 650 needs_session = False 651 652 self.accepts_list = accepts_list 653 """ 654 A list of contexts which will be accepted by this user-mode 655 processor. 656 """ 657 658 omero.util.Servant.__init__(self, ctx, needs_session = needs_session) 659 if cfg is None: 660 self.cfg = os.path.join(omero_home, "etc", "ice.config") 661 self.cfg = os.path.abspath(self.cfg) 662 else: 663 self.cfg = cfg 664 665 # Keep this session alive until the processor is finished 666 self.resources.add( UseSessionHolder(use_session) )
667
668 - def setProxy(self, prx):
669 """ 670 Overrides the default action in order to register this proxy 671 with the session's sharedResources to register for callbacks. 672 The on_newsession handler will also keep new sessions informed. 673 674 See ticket:2304 675 """ 676 omero.util.Servant.setProxy(self, prx) 677 session = self.internal_session() 678 self.register_session(session) 679 680 # Keep other session informed 681 self.ctx.on_newsession = self.register_session
682
683 - def user_client(self, agent):
684 """ 685 Creates an omero.client instance for use by 686 users. 687 """ 688 args = ["--Ice.Config=%s" % (self.cfg)] 689 rtr = self.internal_session().ice_getRouter() 690 if rtr: 691 args.insert(0, "--Ice.Default.Router=%s" % rtr) # FIXME : How do we find an internal router? 692 client = omero.client(args) 693 client.setAgent(agent) 694 return client
695
696 - def internal_session(self):
697 """ 698 Returns the session which should be used for lookups by this instance. 699 Some methods will create a session based on the session parameter. 700 In these cases, the session will belong to the user who is running a 701 script. 702 """ 703 if self.use_session: 704 return self.use_session 705 else: 706 return self.ctx.getSession()
707
708 - def register_session(self, session):
709 self.logger.info("Registering processor %s", self.prx) 710 prx = omero.grid.ProcessorPrx.uncheckedCast(self.prx) 711 session.sharedResources().addProcessor(prx)
712
713 - def lookup(self, job):
714 sf = self.internal_session() 715 gid = job.details.group.id.val 716 handle = WithGroup(sf.createJobHandle(), gid) 717 try: 718 handle.attach(job.id.val) 719 if handle.jobFinished(): 720 handle.close() 721 raise omero.ApiUsageException("Job already finished.") 722 723 prx = WithGroup(sf.getScriptService(), gid) 724 file = prx.validateScript(job, self.accepts_list) 725 726 except omero.SecurityViolation, sv: 727 self.logger.debug("SecurityViolation on validate job %s from group %s", job.id.val, gid) 728 file = None 729 730 return file, handle
731 732 @remoted
733 - def willAccept(self, userContext, groupContext, scriptContext, cb, current = None):
734 735 userID = None 736 if userContext != None: 737 userID = userContext.id.val 738 739 groupID = None 740 if groupContext != None: 741 groupID = groupContext.id.val 742 743 scriptID = None 744 if scriptContext != None: 745 scriptID = scriptContext.id.val 746 747 if scriptID: 748 try: 749 file, handle = self.lookup(scriptContext) 750 handle.close() 751 valid = (file is not None) 752 except: 753 self.logger.error("File lookup failed: user=%s, group=%s, script=%s",\ 754 userID, groupID, scriptID, exc_info=1) 755 return # EARlY EXIT ! 756 else: 757 valid = False 758 for x in self.accepts_list: 759 if isinstance(x, omero.model.Experimenter) and x.id.val == userID: 760 valid = True 761 elif isinstance(x, omero.model.ExperimenterGroup) and x.id.val == groupID: 762 valid = True 763 764 self.logger.debug("Accepts called on: user:%s group:%s scriptjob:%s - Valid: %s", 765 userID, groupID, scriptID, valid) 766 767 try: 768 id = self.internal_session().ice_getIdentity().name 769 cb = cb.ice_oneway() 770 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb) 771 cb.isAccepted(valid, id, str(self.prx)) 772 except exceptions.Exception, e: 773 self.logger.warn("callback failed on willAccept: %s Exception:%s", cb, e) 774 775 return valid
776 777 @remoted
778 - def requestRunning(self, cb, current = None):
779 780 try: 781 cb = cb.ice_oneway() 782 cb = omero.grid.ProcessorCallbackPrx.uncheckedCast(cb) 783 servants = list(self.ctx.servant_map.values()) 784 rv = [] 785 for x in servants: 786 if hasattr(x, "properties"): 787 rv.append(long(x)) 788 cb.responseRunning(rv) 789 except exceptions.Exception, e: 790 self.logger.warn("callback failed on requestRunning: %s Exception:%s", cb, e)
791 792 793 @remoted
794 - def parseJob(self, session, job, current = None):
795 self.logger.info("parseJob: Session = %s, JobId = %s" % (session, job.id.val)) 796 client = self.user_client("OMERO.parseJob") 797 798 try: 799 iskill = False 800 client.joinSession(session).detachOnDestroy() 801 properties = {} 802 properties["omero.scripts.parse"] = "true" 803 prx, process = self.process(client, session, job, current, None, properties, iskill) 804 process.wait() 805 rv = client.getOutput("omero.scripts.parse") 806 if rv != None: 807 return rv.val 808 else: 809 self.logger.warning("No output found for omero.scripts.parse. Keys: %s" % client.getOutputKeys()) 810 return None 811 finally: 812 client.closeSession() 813 del client
814 815 @remoted
816 - def processJob(self, session, params, job, current = None):
817 """ 818 """ 819 self.logger.info("processJob: Session = %s, JobId = %s" % (session, job.id.val)) 820 client = self.user_client("OMERO.processJob") 821 try: 822 client.joinSession(session).detachOnDestroy() 823 prx, process = self.process(client, session, job, current, params, iskill = True) 824 return prx 825 finally: 826 client.closeSession() 827 del client
828 829 830 @perf
831 - def process(self, client, session, job, current, params, properties = None, iskill = True):
832 """ 833 session: session uuid, used primarily if client is None 834 client: an omero.client object which should be attached to a session 835 """ 836 837 if properties is None: properties = {} 838 839 if not session or not job or not job.id: 840 raise omero.ApiUsageException("No null arguments") 841 842 file, handle = self.lookup(job) 843 844 try: 845 if not file: 846 raise omero.ApiUsageException(\ 847 None, None, "Job should have one executable file attached.") 848 849 sf = self.internal_session() 850 if params: 851 self.logger.debug("Checking params for job %s" % job.id.val) 852 svc = sf.getSessionService() 853 inputs = svc.getInputs(session) 854 errors = omero.scripts.validate_inputs(params, inputs, svc, session) 855 if errors: 856 errors = "Invalid parameters:\n%s" % errors 857 raise omero.ValidationException(None, None, errors) 858 859 properties["omero.job"] = str(job.id.val) 860 properties["omero.user"] = session 861 properties["omero.pass"] = session 862 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router") 863 864 process = ProcessI(self.ctx, sys.executable, properties, params, iskill, omero_home = self.omero_home) 865 self.resources.add(process) 866 867 # client.download(file, str(process.script_path)) 868 scriptText = sf.getScriptService().getScriptText(file.id.val) 869 process.script_path.write_bytes(scriptText) 870 871 self.logger.info("Downloaded file: %s" % file.id.val) 872 s = client.sha1(str(process.script_path)) 873 if not s == file.sha1.val: 874 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s) 875 self.logger.error(msg) 876 process.cleanup() 877 raise omero.InternalException(None, None, msg) 878 else: 879 process.activate() 880 handle.setStatus("Running") 881 882 prx = self.ctx.add_servant(current, process) 883 return omero.grid.ProcessPrx.uncheckedCast(prx), process 884 885 finally: 886 handle.close()
887
888 -def usermode_processor(client, serverid = "UsermodeProcessor",\ 889 cfg = None, accepts_list = None, stop_event = None,\ 890 omero_home = path.getcwd()):
891 """ 892 Creates an activates a usermode processor for the given client. 893 It is the responsibility of the client to call "cleanup()" on 894 the ProcessorI implementation which is returned. 895 896 cfg is the path to an --Ice.Config-valid file or files. If none 897 is given, the value of ICE_CONFIG will be taken from the environment 898 if available. Otherwise, all properties will be taken from the client 899 instance. 900 901 accepts_list is the list of IObject instances which will be passed to 902 omero.api.IScripts.validateScript. If none is given, only the current 903 Experimenter's own object will be passed. 904 905 stop_event is an threading.Event. One will be acquired from 906 omero.util.concurrency.get_event if none is provided. 907 """ 908 909 if cfg is None: 910 cfg = os.environ.get("ICE_CONFIG") 911 912 if accepts_list is None: 913 uid = client.sf.getAdminService().getEventContext().userId 914 accepts_list = [omero.model.ExperimenterI(uid, False)] 915 916 if stop_event is None: 917 stop_event = omero.util.concurrency.get_event(name="UsermodeProcessor") 918 919 ctx = omero.util.ServerContext(serverid, client.ic, stop_event) 920 impl = omero.processor.ProcessorI(ctx, 921 use_session=client.sf, accepts_list=accepts_list, cfg=cfg, 922 omero_home = omero_home) 923 ctx.add_servant(client.adapter, impl) 924 return impl
925