Package omero :: Module processor
[hide private]
[frames] | no frames]

Source Code for Module omero.processor

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Grid Processor 
  4  # Copyright 2008 Glencoe Software, Inc.  All Rights Reserved. 
  5  # Use is subject to license terms supplied in LICENSE.txt 
  6  # 
  7   
  8  import os 
  9  import sys 
 10  import time 
 11  import signal 
 12  import logging 
 13  import traceback 
 14  import exceptions 
 15  import killableprocess as subprocess 
 16   
 17  from path import path 
 18   
 19  import Ice 
 20  import omero 
 21  import omero.clients 
 22  import omero.util 
 23  import omero.util.concurrency 
 24   
 25  from omero.util.temp_files import create_path, remove_path 
 26  from omero.util.decorators import remoted, perf, locked 
 27  from omero.rtypes import * 
28 29 30 -class ProcessI(omero.grid.Process, omero.util.SimpleServant):
31 """ 32 Wrapper around a subprocess.Popen instance. Returned by ProcessorI 33 when a job is submitted. This implementation uses the given 34 interpreter to call a file that must be named "script" in the 35 generated temporary directory. 36 37 Call is equivalent to: 38 39 cd TMP_DIR 40 ICE_CONFIG=./config interpreter ./script >out 2>err & 41 42 The properties argument is used to generate the ./config file. 43 44 The params argument may be null in which case this process 45 is being used solely to calculate the parameters for the script 46 ("omero.scripts.parse=true") 47 48 If iskill is True, then on cleanup, this process will reap the 49 attached session completely. 50 """ 51
52 - def __init__(self, ctx, interpreter, properties, params, iskill = False,\ 53 Popen = subprocess.Popen, callback_cast = omero.grid.ProcessCallbackPrx.uncheckedCast):
54 """ 55 Popen and callback_Cast are primarily for testing. 56 """ 57 omero.util.SimpleServant.__init__(self, ctx) 58 self.interpreter = interpreter #: Executable which will be used on the script 59 self.properties = properties #: Properties used to create an Ice.Config 60 self.params = params #: JobParams for this script. Possibly None 61 self.iskill = iskill #: Whether or not, cleanup should kill the session 62 self.Popen = Popen #: Function which should be used for creating processes 63 self.callback_cast = callback_cast #: Function used to cast all ProcessCallback proxies 64 # Non arguments (mutable state) 65 self.rcode = None #: return code from popen 66 self.callbacks = {} #: dictionary from id strings to callback proxies 67 self.popen = None #: process. if None, then this instance isn't alive. 68 self.pid = None #: pid of the process. Once set, isn't nulled. 69 self.started = None #: time the process started 70 self.stopped = None #: time of deactivation 71 # Non arguments (immutable state) 72 self.uuid = properties["omero.user"] #: session this instance is tied to 73 74 # More fields set by these methods 75 self.make_files() 76 self.make_env() 77 self.make_config() 78 self.logger.info("Created %s in %s" % (self.uuid, self.dir))
79 80 # 81 # Initialization methods 82 # 83
84 - def make_env(self):
85 self.env = omero.util.Environment("PATH", "PYTHONPATH",\ 86 "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MLABRAW_CMD_STR") 87 # WORKAROUND 88 # Currently duplicating the logic here as in the PYTHONPATH 89 # setting of the grid application descriptor (see etc/grid/*.xml) 90 # This should actually be taken care of in the descriptor itself 91 # by having setting PYTHONPATH to an absolute value. This is 92 # not currently possible with IceGrid (without using icepatch -- 93 # see 39.17.2 "node.datadir). 94 self.env.append("PYTHONPATH", str(path.getcwd() / "lib" / "python")) 95 self.env.set("ICE_CONFIG", str(self.config_path))
96
97 - def make_files(self):
98 self.dir = create_path("process", ".dir", folder = True) 99 self.script_path = self.dir / "script" 100 self.config_path = self.dir / "config" 101 self.stdout_path = self.dir / "out" 102 self.stderr_path = self.dir / "err"
103
104 - def make_config(self):
105 """ 106 Creates the ICE_CONFIG file used by the client. 107 """ 108 config_file = open(str(self.config_path), "w") 109 try: 110 for key in self.properties.iterkeys(): 111 config_file.write("%s=%s\n"%(key, self.properties[key])) 112 finally: 113 config_file.close()
114
115 - def tmp_client(self):
116 """ 117 Create a client for performing cleanup operations. 118 This client should be closed as soon as possible 119 by the process 120 """ 121 try: 122 client = omero.client(["--Ice.Config=%s" % str(self.config_path)]) 123 client.createSession().detachOnDestroy() 124 self.logger.debug("client: %s" % client.sf) 125 return client 126 except: 127 self.logger.error("Failed to create client for %s" % self.uuid) 128 return None
129 130 # 131 # Activation / Deactivation 132 # 133 134 @locked
135 - def activate(self):
136 """ 137 Process creation has to wait until all external downloads, etc 138 are finished. 139 """ 140 141 if self.isActive(): 142 raise omero.ApiUsageException(None, None, "Already activated") 143 144 self.stdout = open(str(self.stdout_path), "w") 145 self.stderr = open(str(self.stderr_path), "w") 146 self.popen = self.Popen([self.interpreter, "./script"], cwd=str(self.dir), env=self.env(), stdout=self.stdout, stderr=self.stderr) 147 self.pid = self.popen.pid 148 self.started = time.time() 149 self.stopped = None 150 self.status("Activated")
151 152 @locked
153 - def deactivate(self):
154 """ 155 Cleans up the temporary directory used by the process, and terminates 156 the Popen process if running. 157 """ 158 159 if not self.isActive(): 160 raise omero.ApiUsageException(None, None, "Not active") 161 162 if self.stopped: 163 # Prevent recursion since we are reusing kill & cancel 164 return 165 166 self.stopped = time.time() 167 d_start = time.time() 168 self.status("Deactivating") 169 170 # None of these should throw, but just in case 171 try: 172 173 self.shutdown() # Calls cancel & kill which recall this method! 174 self.popen = None # Now we are finished 175 176 self.cleanup_output() 177 self.upload_output() # Important! 178 self.cleanup_tmpdir() 179 180 except exceptions.Exception: 181 self.logger.error("FAILED TO CLEANUP pid=%s (%s)", self.pid, self.uuid, exc_info = True) 182 183 d_stop = time.time() 184 elapsed = int(self.stopped - self.started) 185 d_elapsed = int(d_stop - d_start) 186 self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
187 188 @locked
189 - def isActive(self):
190 """ 191 Tests only if this instance has a non-None popen attribute. After activation 192 this method will return True until the popen itself returns a non-None 193 value (self.rcode) at which time it will be nulled and this method will again 194 return False 195 """ 196 return self.popen is not None
197 198 @locked
199 - def wasActivated(self):
200 """ 201 Returns true only if this instance has either a non-null 202 popen or a non-null rcode field. 203 """ 204 return self.popen is not None or self.rcode is not None
205 206 @locked
207 - def isRunning(self):
208 return self.popen is not None and self.rcode is None
209 210 @locked
211 - def isFinished(self):
212 return self.rcode is not None
213 214 @locked
215 - def alreadyDone(self):
216 """ 217 Allows short-cutting various checks if we already 218 have a rcode for this popen. A non-None return value 219 implies that a process was started and returned 220 the given non-None value itself. 221 """ 222 if not self.wasActivated: 223 raise omero.InternalException(None, None, "Process never activated") 224 return self.isFinished()
225 226 # 227 # Cleanup methods 228 # 229
230 - def __del__(self):
231 self.cleanup()
232 233 @perf 234 @locked
235 - def check(self):
236 """ 237 Called periodically to keep the session alive. Returns 238 False if this resource can be cleaned up. (Resources API) 239 """ 240 241 if not self.wasActivated(): 242 return True # This should only happen on startup, so ignore 243 244 try: 245 self.poll() 246 self.ctx.getSession().getSessionService().getSession(self.uuid) 247 return True 248 except: 249 self.status("Keep alive failed") 250 return False
251 252 @perf 253 @locked
254 - def cleanup(self):
255 """ 256 Deactivates the process (if active) and cleanups the server 257 connection. (Resources API) 258 """ 259 260 if self.isRunning(): 261 self.deactivate() 262 263 try: 264 sf = self.ctx.getSession(recreate = False) 265 except: 266 self.logger.warn("Can't get session for cleanup") 267 return 268 269 self.status("Cleaning") 270 svc = sf.getSessionService() 271 obj = omero.model.SessionI() 272 obj.uuid = omero.rtypes.rstring(self.uuid) 273 try: 274 if self.iskill: 275 self.status("Killing session") 276 while svc.closeSession(obj) > 0: 277 pass 278 # No action to be taken when iskill == False if 279 # we don't have an actual client to worry with. 280 except: 281 self.logger.error("Error on session cleanup, kill=%s" % self.iskill, exc_info = True)
282
283 - def cleanup_output(self):
284 """ 285 Flush and close the stderr and stdout streams. 286 """ 287 try: 288 if hasattr(self, "stderr"): 289 self.stderr.flush() 290 self.stderr.close() 291 except: 292 self.logger.error("cleanup of sterr failed", exc_info = True) 293 try: 294 if hasattr(self, "stdout"): 295 self.stdout.flush() 296 self.stdout.close() 297 except: 298 self.logger.error("cleanup of sterr failed", exc_info = True)
299
300 - def upload_output(self):
301 """ 302 If this is not a params calculation (i.e. parms != null) and the 303 stdout or stderr are non-null, they they will be uploaded and 304 attached to the job. 305 """ 306 client = self.tmp_client() 307 if not client: 308 self.logger.error("No client: Cannot upload output for pid=%s (%s)", self.pid, self.uuid) 309 return 310 311 try: 312 if self.params: 313 out_format = self.params.stdoutFormat 314 err_format = self.params.stderrFormat 315 upload = True 316 else: 317 out_format = "text/plain" 318 err_format = out_format 319 upload = False 320 321 self._upload(upload, client, self.stdout_path, "stdout", out_format) 322 self._upload(upload, client, self.stderr_path, "stderr", err_format) 323 finally: 324 client.__del__() # Safe closeSession
325
326 - def _upload(self, upload, client, filename, name, format):
327 328 if not upload or not format: 329 return 330 331 filename = str(filename) # Might be path.path 332 sz = os.path.getsize(filename) 333 if not sz: 334 self.logger.info("No %s" % name) 335 return 336 337 try: 338 ofile = client.upload(filename, name=name, type=format) 339 jobid = long(client.getProperty("omero.job")) 340 link = omero.model.JobOriginalFileLinkI() 341 link.parent = omero.model.ScriptJobI(rlong(jobid), False) 342 link.child = ofile 343 client.getSession().getUpdateService().saveObject(link) 344 self.status("Uploaded %s bytes of %s to %s" % (sz, filename, ofile.id.val)) 345 except: 346 self.logger.error("Error on upload of %s for pid=%s (%s)", filename, self.pid, self.uuid, exc_info = True)
347
348 - def cleanup_tmpdir(self):
349 """ 350 Remove all known files and finally the temporary directory. 351 If other files exist, an exception will be raised. 352 """ 353 try: 354 remove_path(self.dir) 355 except: 356 self.logger.error("Failed to remove dir %s" % self.dir, exc_info = True)
357 358 # 359 # popen methods 360 # 361
362 - def status(self, msg = ""):
363 if self.isRunning(): 364 self.rcode = self.popen.poll() 365 self.logger.info("%s : %s", self, msg)
366 367 @perf 368 @remoted
369 - def poll(self, current = None):
370 """ 371 Checks popen.poll() (if active) and notifies all callbacks 372 if necessary. If this method returns a non-None value, then 373 the process will be marked inactive. 374 """ 375 376 if self.alreadyDone(): 377 return rint(self.rcode) 378 379 self.status("Polling") 380 if self.rcode is None: 381 # Haven't finished yet, so do nothing. 382 return None 383 else: 384 self.deactivate() 385 rv = rint(self.rcode) 386 self.allcallbacks("processFinished", rv) 387 return rv
388 389 @perf 390 @remoted
391 - def wait(self, current = None):
392 """ 393 Waits on popen.wait() to return (if active) and notifies 394 all callbacks. Marks this process as inactive. 395 """ 396 397 if self.alreadyDone(): 398 return self.rcode 399 400 self.status("Waiting") 401 self.rcode = self.popen.wait() 402 self.deactivate() 403 self.allcallbacks("processFinished", self.rcode) 404 return self.rcode
405
406 - def _term(self):
407 """ 408 Attempts to cancel the process by sending SIGTERM 409 (or similar) 410 """ 411 try: 412 self.status("os.kill(TERM)") 413 os.kill(self.popen.pid, signal.SIGTERM) 414 except AttributeError: 415 self.logger.debug("No os.kill(TERM). Skipping cancel")
416
417 - def _send(self, iskill):
418 """ 419 Helper method for sending signals. This method only 420 makes a call is the process is active. 421 """ 422 if self.isRunning(): 423 try: 424 if self.popen.poll() is None: 425 if iskill: 426 self.status("popen.kill(True)") 427 self.popen.kill(True) 428 else: 429 self._term() 430 431 else: 432 self.status("Skipped signal") 433 except OSError, oserr: 434 self.logger.debug("err on pid=%s iskill=%s : %s", self.popen.pid, iskill, oserr)
435 436 @perf 437 @remoted
438 - def cancel(self, current = None):
439 """ 440 Tries to cancel popen (if active) and notifies callbacks. 441 """ 442 443 if self.alreadyDone(): 444 return True 445 446 self._send(iskill=False) 447 finished = self.isFinished() 448 if finished: 449 self.deactivate() 450 self.allcallbacks("processCancelled", finished) 451 return finished
452 453 @perf 454 @remoted
455 - def kill(self, current = None):
456 457 if self.alreadyDone(): 458 return True 459 460 self._send(iskill=True) 461 finished = self.isFinished() 462 if finished: 463 self.deactivate() 464 self.allcallbacks("processKilled", finished) 465 return finished
466 467 @perf 468 @remoted
469 - def shutdown(self, current = None):
470 """ 471 If self.popen is active, then first call cancel, wait a period of 472 time, and finally call kill. 473 """ 474 475 if self.alreadyDone(): 476 return 477 478 self.status("Shutdown") 479 try: 480 for i in range(5, 0, -1): 481 if self.cancel(): 482 break 483 else: 484 self.logger.warning("Shutdown: %s (%s). Killing in %s seconds.", self.pid, self.uuid, 6*(i-1)+1) 485 self.stop_event.wait(6) 486 self.kill() 487 except: 488 self.logger.error("Shutdown failed: %s (%s)", self.pid, self.uuid, exc_info = True)
489 490 # 491 # Callbacks 492 # 493 494 @remoted 495 @locked
496 - def registerCallback(self, callback, current = None):
497 try: 498 id = callback.ice_getIdentity() 499 key = "%s/%s" % (id.category, id.name) 500 callback = callback.ice_oneway() 501 callback = self.callback_cast(callback) 502 if not callback: 503 e = "Callback is invalid" 504 else: 505 self.callbacks[key] = callback 506 self.logger.debug("Added callback: %s", key) 507 return 508 except exceptions.Exception, ex: 509 e = ex 510 # Only reached on failure 511 msg = "Failed to add callback: %s. Reason: %s" % (callback, e) 512 self.logger.debug(msg) 513 raise omero.ApiUsageException(None, None, msg)
514 515 @remoted 516 @locked
517 - def unregisterCallback(self, callback, current = None):
518 try: 519 id = callback.ice_getIdentity() 520 key = "%s/%s" % (id.category, id.name) 521 if not key in self.callback: 522 raise omero.ApiUsageException(None, None, "No callback registered with id: %s" % key) 523 del self.callbacks[key] 524 self.logger.debug("Removed callback: %s", key) 525 except exceptions.Exception, e: 526 msg = "Failed to remove callback: %s. Reason: %s" % (callback, e) 527 self.logger.debug(msg) 528 raise omero.ApiUsageException(None, None, msg)
529 530 @locked
531 - def allcallbacks(self, method, arg):
532 self.status("Callback %s" % method) 533 for key, cb in self.callbacks.items(): 534 try: 535 m = getattr(cb, method) 536 m(arg) 537 except: 538 self.logger.error("Error calling callback %s on pid=%s (%s)" % (key, self.pid, self.uuid), exc_info = True)
539
540 - def __str__(self):
541 return "<proc:%s,rc=%s,uuid=%s>" % (self.pid, self.rcode, self.uuid)
542
543 -class ProcessorI(omero.grid.Processor, omero.util.Servant):
544
545 - def __init__(self, ctx):
546 omero.util.Servant.__init__(self, ctx, needs_session = True) 547 self.cfg = os.path.join(os.curdir, "etc", "ice.config") 548 self.cfg = os.path.abspath(self.cfg)
549 550 @remoted
551 - def parseJob(self, session, job, current = None):
552 553 client = omero.client(["--Ice.Config=%s" % (self.cfg)]) 554 try: 555 client.joinSession(session).detachOnDestroy() 556 return self.parse(client, session, job, current, iskill = False) 557 finally: 558 client.closeSession() 559 del client
560 561 @remoted
562 - def processJob(self, session, job, current = None):
563 """ 564 """ 565 client = omero.client(["--Ice.Config=%s" % (self.cfg)]) 566 try: 567 client.joinSession(session).detachOnDestroy() 568 params = self.parse(client, session, job, current, iskill = False) 569 return self.process(client, session, job, current, params, iskill = True) 570 finally: 571 client.closeSession() 572 del client
573 574 @perf
575 - def parse(self, client, session, job, current, iskill):
576 577 if not session or not job or not job.id: 578 raise omero.ApiUsageException("No null arguments") 579 580 self.logger.info("parseJob: Session = %s, JobId = %s" % (session, job.id.val)) 581 properties = {} 582 properties["omero.scripts.parse"] = "true" 583 process = self.process(client, session, job, current, None, properties, iskill) 584 process.wait() 585 rv = client.getOutput("omero.scripts.parse") 586 if rv != None: 587 return rv.val 588 else: 589 self.logger.warning("No output found for omero.scripts.parse. Keys: %s" % client.getOutputKeys()) 590 return None
591 592 @perf
593 - def process(self, client, session, job, current, params, properties = {}, iskill = True):
594 """ 595 session: session uuid, used primarily if client is None 596 client: an omero.client object which should be attached to a session 597 """ 598 599 sf = client.getSession() 600 handle = sf.createJobHandle() 601 handle.attach(job.id.val) 602 if handle.jobFinished(): 603 raise omero.ApiUsageException("Job already finished.") 604 605 file = sf.getQueryService().findByQuery(\ 606 """select o from Job j 607 join j.originalFileLinks links 608 join links.child o 609 join o.format 610 where 611 j.id = %d 612 and o.details.owner.id = 0 613 and o.format.value = 'text/x-python' 614 """ % job.id.val, None) 615 616 617 if not file: 618 raise omero.ApiUsageException(\ 619 None, None, "Job should have one executable file attached.") 620 621 properties["omero.job"] = str(job.id.val) 622 properties["omero.user"] = session 623 properties["omero.pass"] = session 624 properties["Ice.Default.Router"] = client.getProperty("Ice.Default.Router") 625 626 self.logger.info("processJob: Session = %s, JobId = %s" % (session, job.id.val)) 627 process = ProcessI(self.ctx, "python", properties, params, iskill) 628 self.resources.add(process) 629 client.download(file, str(process.script_path)) 630 self.logger.info("Downloaded file: %s" % file.id.val) 631 s = client.sha1(str(process.script_path)) 632 if not s == file.sha1.val: 633 msg = "Sha1s don't match! expected %s, found %s" % (file.sha1.val, s) 634 self.logger.error(msg) 635 process.cleanup() 636 raise omero.InternalException(None, None, msg) 637 else: 638 process.activate() 639 640 prx = current.adapter.addWithUUID(process) 641 return omero.grid.ProcessPrx.uncheckedCast(prx)
642