Package omero :: Package util
[hide private]
[frames] | no frames]

Source Code for Package omero.util

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Utilities package 
  4  # 
  5  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  6  # Use is subject to license terms supplied in LICENSE.txt 
  7  # 
  8   
  9  import os 
 10  import sys 
 11  import Ice 
 12  import path 
 13  import time 
 14  import uuid 
 15  import omero 
 16  import IcePy 
 17  import IceGrid 
 18  import logging 
 19  import platform 
 20  import Glacier2 
 21  import threading 
 22  import exceptions 
 23  import logging.handlers 
 24  import omero.util.concurrency 
 25   
 26  from omero.util.decorators import locked 
 27   
 28  LOGDIR = os.path.join("var","log") 
 29  LOGFORMAT =  """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s""" 
 30  LOGLEVEL = logging.INFO 
 31  LOGSIZE = 500000000 
 32  LOGNUM = 9 
 33  LOGMODE = "a" 
 34   
 35  orig_stdout = sys.stdout 
 36  orig_stderr = sys.stderr 
37 38 -def make_logname(self):
39 """ 40 Generates a logname from the given instance using the module and name from its class 41 """ 42 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__) 43 return log_name
44
45 -def configure_logging(logdir = None, logfile = None, loglevel = LOGLEVEL,\ 46 format = LOGFORMAT, filemode = LOGMODE, maxBytes = LOGSIZE, backupCount = LOGNUM, time_rollover = False):
47 48 if logdir is None or logfile is None: 49 handler = logging.StreamHandler() 50 elif not time_rollover: 51 handler = logging.handlers.RotatingFileHandler(os.path.join(logdir, logfile), maxBytes = maxBytes, backupCount = backupCount) 52 else: 53 handler = logging.handlers.TimedRotatingFileHandler(os.path.join(logdir, logfile),'midnight',1) 54 # Windows will not allow renaming (or deleting) a file that's open. 55 # There's nothing the logging package can do about that. 56 try: 57 sys.getwindowsversion() 58 except: 59 handler.doRollover() 60 61 handler.setLevel(loglevel) 62 formatter = logging.Formatter(format) 63 handler.setFormatter(formatter) 64 rootLogger = logging.getLogger() 65 rootLogger.setLevel(loglevel) 66 rootLogger.addHandler(handler) 67 return rootLogger
68
69 -def configure_server_logging(props):
70 """ 71 Takes an Ice.Properties instance and configures logging 72 """ 73 program_name = props.getProperty("Ice.Admin.ServerId") 74 # Using Ice.ProgramName on Windows failed 75 log_name = program_name+".log" 76 log_timed = props.getPropertyWithDefault("omero.logging.timedlog","False")[0] in ('T', 't') 77 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 78 log_size = int(props.getPropertyWithDefault("omero.logging.logsize",str(LOGSIZE))) 79 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 80 log_level = int(props.getPropertyWithDefault("omero.logging.level",str(LOGLEVEL))) 81 configure_logging(LOGDIR, log_name, loglevel=log_level, maxBytes=log_size, backupCount=log_num, time_rollover = log_timed) 82 83 sys.stdout = StreamRedirect(logging.getLogger("stdout")) 84 sys.stderr = StreamRedirect(logging.getLogger("stderr"))
85
86 -class StreamRedirect(object):
87 """ 88 Since all server components should exclusively using the logging module 89 any output to stdout or stderr is caught and logged at "WARN". This is 90 useful, especially in the case of Windows, where stdout/stderr is eaten. 91 """ 92
93 - def __init__(self, logger):
94 self.logger = logger 95 self.internal = logging.getLogger("StreamRedirect") 96 self.softspace = False
97
98 - def flush(self):
99 pass
100
101 - def write(self, msg):
102 msg = msg.strip() 103 if msg: 104 self.logger.warn(msg)
105
106 - def __getattr__(self, name):
107 self.internal.warn("No attribute: %s" % name)
108
109 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
110 """ 111 Try to return a ServiceFactory from the grid. 112 113 Try a number of times then give up and raise the 114 last exception returned. This method will only 115 work internally to the grid, i.e. behind the Glacier2 116 firewall. It is intended for internal servers to 117 be able to create sessions for accessing the database. :: 118 communicator := Ice.Communicator used to find the registry 119 user := Username which should have a session created 120 group := Group into which the session should be logged 121 retries := Number of session creation retries before throwing 122 interval := Seconds between retries 123 client_uuid := Uuid of the client which should be used 124 """ 125 log = logging.getLogger("omero.utils") 126 if stop_event == None: 127 stop_event = omero.util.concurrency.get_event() 128 129 tryCount = 0 130 excpt = None 131 query = communicator.stringToProxy("IceGrid/Query") 132 query = IceGrid.QueryPrx.checkedCast(query) 133 134 import omero_Constants_ice 135 implicit_ctx = communicator.getImplicitContext() 136 if client_uuid is not None: 137 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid) 138 else: 139 if not implicit_ctx.containsKey(omero.constants.CLIENTUUID): 140 client_uuid = str(uuid.uuid4()) 141 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid) 142 143 while tryCount < retries: 144 if stop_event.isSet(): # Something is shutting down, exit. 145 return None 146 try: 147 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0] 148 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz) 149 sf = blitz.create(user, None) 150 # Group currently unused. 151 return omero.api.ServiceFactoryPrx.checkedCast(sf) 152 except Exception, e: 153 tryCount += 1 154 log.info("Failed to get session on attempt %s", str(tryCount)) 155 excpt = e 156 stop_event.wait(interval) 157 158 log.warn("Reason: %s", str(excpt)) 159 raise excpt
160
161 -def create_admin_session(communicator):
162 """ 163 """ 164 reg = communicator.stringToProxy("IceGrid/Registry") 165 reg = IceGrid.RegistryPrx.checkedCast(reg) 166 adm = reg.createAdminSession('null', '') 167 return adm
168
169 -def add_grid_object(communicator, obj):
170 """ 171 """ 172 sid = communicator.identityToString(obj.ice_getIdentity()) 173 adm = create_admin_session(communicator) 174 prx = adm.getAdmin() 175 try: 176 try: 177 prx.addObject(obj) 178 except IceGrid.ObjectExistsException: 179 prx.updateObject(obj) 180 finally: 181 adm.destroy()
182
183 -def long_to_path(id, root=""):
184 """ 185 Converts a long to a path such that for all directiories only 186 a 1000 files and a 1000 subdirectories will be returned. 187 188 This method duplicates the logic in 189 ome.io.nio.AbstractFileSystemService.java:getPath() 190 """ 191 suffix = "" 192 remaining = id 193 dirno = 0 194 195 if id is None or id == "": 196 raise exceptions.Exception("Expecting a not-null id.") 197 198 id = long(id) 199 200 if id < 0: 201 raise exceptions.Exception("Expecting a non-negative id.") 202 203 while (remaining > 999): 204 remaining /= 1000 205 206 if remaining > 0: 207 dirno = remaining % 1000 208 suffix = os.path.join("Dir-%03d" % dirno, suffix) 209 210 return os.path.join(root, "%s%s" %(suffix,id))
211
212 -class ServerContext(object):
213 """ 214 Context passed to all servants. 215 216 server_id, communicator, and stop_event will be 217 constructed by the top-level Server instance. 218 219 A context instance may also be configured to hold 220 on to an internal session (ServiceFactoryPrx) and 221 keep it alive. 222 223 This instance obeys the Resources API and calls 224 sf.keepAlive(None) on every check call, but does 225 nothing on cleanup. The sf instance must be manually 226 cleaned as the final operation of a servant. 227 228 (Note: cleanup of the server context indicates 229 server shutdown, so should be infrequent) 230 """ 231
232 - def __init__(self, server_id, communicator, stop_event, on_newsession = None):
233 self._lock = threading.RLock() 234 self.logger = logging.getLogger("omero.util.ServerContext") 235 self.server_id = server_id 236 self.communicator = communicator 237 self.stop_event = stop_event 238 self.servant_map = dict() 239 self.on_newsession = None
240 241 @locked
242 - def add_servant(self, adapter_or_current, servant, ice_identity = None):
243 oa = adapter_or_current 244 if isinstance(adapter_or_current, (Ice.Current, IcePy.Current)): 245 oa = oa.adapter 246 if ice_identity is None: 247 prx = oa.addWithUUID(servant) 248 else: 249 prx = oa.add(servant, ice_identity) 250 251 servant.setProxy(prx) 252 self.servant_map[prx] = servant 253 return prx
254
255 - def newSession(self):
256 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event) 257 if callable(self.on_newsession): 258 self.on_newsession(self.session)
259
260 - def hasSession(self):
261 return hasattr(self, "session")
262 263 @locked
264 - def getSession(self, recreate = True):
265 """ 266 Returns the ServiceFactoryPrx configured for the context if 267 available. If the context was not configured for sessions, 268 an ApiUsageException will be thrown: servants should know 269 whether or not they were configured for sessions. 270 See Servant(..., needs_session = True) 271 272 Otherwise, if there is no ServiceFactoryPrx, an attempt will 273 be made to create one if recreate == True. If the value is None 274 or non can be recreated, an InternalException will be thrown. 275 276 TODO : currently no arguments are provided for re-creating these, 277 but also not in Servant.__init__ 278 """ 279 if not self.hasSession(): 280 raise omero.ApiUsageException("Not configured for server connection") 281 282 if self.session: 283 try: 284 self.session.keepAlive(None) 285 except Ice.CommunicatorDestroyedException: 286 self.session = None # Ignore 287 except exceptions.Exception, e: 288 self.logger.warn("Connection failure: %s" % e) 289 self.session = None 290 291 if self.session is None and recreate: 292 try: 293 self.newSession() 294 self.logger.info("Established connection: %s" % self.session) 295 except exceptions.Exception, e: 296 self.logger.warn("Failed to establish connection: %s" % e) 297 298 if self.session is None: 299 raise omero.InternalException("No connection to server") 300 301 return self.session
302
303 - def check(self):
304 """ 305 Calls getSession() but always returns True. This keeps the context 306 available in the resources for later uses, and tries to re-establish 307 a connection in case Blitz goes down. 308 """ 309 try: 310 self.getSession() 311 except: 312 pass 313 return True
314
315 - def cleanup(self):
316 """ 317 Does nothing. Context clean up must happen manually 318 since later activities may want to reuse it. Servants using 319 a server connection should cleanup the instance *after* Resources 320 is cleaned up 321 """ 322 pass
323
324 -class Server(Ice.Application):
325 """ 326 Basic server implementation which can be used for 327 implementing a standalone python server which can 328 be started from icegridnode. 329 330 The servant implementation MUST have a constructor 331 which takes a single ServerContext argument AND 332 have a cleanup() method 333 334 Logging is configured relative to the current directory 335 to be in var/log by default. 336 337 Usage:: 338 339 if __name__ == "__main__": 340 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services","")) 341 sys.exit(app.main(sys.argv)) 342 343 app.impl now points to an instance of ServicesI 344 345 """ 346
347 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
348 349 self.impl_class = impl_class 350 self.adapter_name = adapter_name 351 self.identity = identity 352 self.logdir = logdir 353 self.stop_event = omero.util.concurrency.get_event()
354
355 - def run(self,args):
356 357 from omero.rtypes import ObjectFactories as rFactories 358 from omero.columns import ObjectFactories as cFactories 359 360 props = self.communicator().getProperties() 361 configure_server_logging(props) 362 363 self.logger = logging.getLogger("omero.util.Server") 364 self.logger.info("*"*80) 365 self.logger.info("Starting") 366 367 self.shutdownOnInterrupt() 368 369 try: 370 371 self.objectfactory = omero.clients.ObjectFactory() 372 self.objectfactory.registerObjectFactory(self.communicator()) 373 for of in rFactories.values() + cFactories.values(): 374 of.register(self.communicator()) 375 376 try: 377 serverid = self.communicator().getProperties().getProperty("Ice.ServerId") 378 ctx = ServerContext(serverid, self.communicator(), self.stop_event) 379 self.impl = self.impl_class(ctx) 380 getattr(self.impl, "cleanup") # Required per docs 381 except: 382 self.logger.error("Failed initialization", exc_info=1) 383 sys.exit(100) 384 385 try: 386 self.adapter = self.communicator().createObjectAdapter(self.adapter_name) 387 self.adapter.activate() 388 ctx.add_servant(self.adapter, self.impl, self.identity) # calls setProxy 389 add_grid_object(self.communicator(), self.impl.prx) # This must happen _after_ activation 390 except: 391 self.logger.error("Failed activation", exc_info=1) 392 sys.exit(200) 393 394 self.logger.info("Entering main loop") 395 self.communicator().waitForShutdown() 396 finally: 397 self.stop_event.set() # Let's all waits shutdown 398 self.logger.info("Cleanup") 399 self.cleanup() 400 self.logger.info("Stopped") 401 self.logger.info("*"*80)
402
403 - def cleanup(self):
404 """ 405 Cleans up all resources that were created by this server. 406 Primarily the one servant instance. 407 """ 408 if hasattr(self,"impl"): 409 try: 410 self.impl.cleanup() 411 finally: 412 del self.impl
413
414 415 -class SimpleServant(object):
416 """ 417 Base servant initialization. Doesn't create or try to cleanup 418 a top-level Resources thread. This is useful for large numbers 419 of servants. For servers and other singleton-like servants, 420 see "Servant" 421 """
422 - def __init__(self, ctx):
423 self._lock = threading.RLock() 424 self.prx = None # Proxy which points to self 425 self.ctx = ctx 426 self.stop_event = ctx.stop_event 427 self.communicator = ctx.communicator 428 self.logger = logging.getLogger(make_logname(self)) 429 self.logger.debug("Created")
430
431 - def setProxy(self, prx):
432 """ 433 Should be overwritten for post-initialization activities. 434 The reason this method exists is that the implementation 435 must be complete before registering it with the adapter. 436 """ 437 self.prx = prx
438
439 -class Servant(SimpleServant):
440 """ 441 Abstract servant which can be used along with a slice2py 442 generated dispatch class as the base type of high-level servants. 443 These provide resource cleanup as per the omero.util.Server 444 class. 445 446 By passing "needs_session = True" to this constructor, an internal 447 session will be created and stored in ServerContext as well as 448 registered with self.resources 449 """ 450
451 - def __init__(self, ctx, needs_session = False):
452 SimpleServant.__init__(self, ctx) 453 self.resources = omero.util.Resources(sleeptime = 60, stop_event = self.stop_event) 454 if needs_session: 455 self.ctx.newSession() 456 self.resources.add(self.ctx)
457
458 - def cleanup(self):
459 """ 460 Cleanups all resoures created by this servant. Calling 461 cleanup multiple times should be safe. 462 """ 463 resources = self.resources 464 self.resources = None 465 if resources != None: 466 self.logger.info("Cleaning up") 467 resources.cleanup() 468 self.logger.info("Done") 469 if self.ctx.hasSession(): 470 try: 471 sf = self.ctx.getSession(recreate=False) 472 self.logger.debug("Destroying %s" % sf) 473 sf.destroy() 474 except: 475 pass
476
477 - def __del__(self):
478 self.cleanup()
479
480 481 -class Resources:
482 """ 483 Container class for storing resources which should be 484 cleaned up on close and periodically checked. Use 485 stop_event.set() to stop the internal thread. 486 """ 487
488 - def __init__(self, sleeptime = 60, stop_event = None):
489 """ 490 Add resources via add(object). They should have a no-arg cleanup() 491 and a check() method. 492 493 The check method will be called periodically (default: 60 seconds) 494 on each resource. The cleanup method will be called on 495 Resources.cleanup() 496 """ 497 498 self._lock = threading.RLock() 499 self.logger = logging.getLogger("omero.util.Resources") 500 self.stop_event = stop_event 501 if not self.stop_event: 502 self.stop_event = omero.util.concurrency.get_event() 503 504 if sleeptime < 5: 505 raise exceptions.Exception("Sleep time should be greater than 5: " % sleeptime) 506 507 self.sleeptime = sleeptime 508 self.stuff = [] 509 510 class Task(threading.Thread): 511 """ 512 Internal thread used for checking "stuff" 513 """ 514 def run(self): 515 ctx = self.ctx # Outer class 516 ctx.logger.info("Starting") 517 while not ctx.stop_event.isSet(): 518 try: 519 ctx.logger.debug("Executing") 520 copy = ctx.copyStuff() 521 remove = ctx.checkAll(copy) 522 ctx.removeAll(remove) 523 except: 524 ctx.logger.error("Exception during execution", exc_info = True) 525 526 ctx.logger.debug("Sleeping %s" % ctx.sleeptime) 527 # ticket:1531 - Attempting to catch threading issues 528 try: 529 ctx.stop_event.wait(ctx.sleeptime) 530 except ValueError: 531 pass 532 533 ctx.logger.info("Halted")
534 535 self.thread = Task() 536 self.thread.ctx = self 537 self.thread.start() 538 539 @locked
540 - def copyStuff(self):
541 """ 542 Within a lock, copy the "stuff" list and reverse it. 543 The list is reversed so that entries added 544 later, which may depend on earlier added entries 545 get a chance to be cleaned up first. 546 """ 547 copy = list(self.stuff) 548 copy.reverse() 549 return copy
550 551 # Not locked
552 - def checkAll(self, copy):
553 """ 554 While stop_event is unset, go through the copy 555 of stuff and call the check method on each 556 entry. Any that throws an exception or returns 557 a False value will be returned in the remove list. 558 """ 559 remove = [] 560 for m in copy: 561 if self.stop_event.isSet(): 562 return # Let cleanup handle this 563 self.logger.debug("Checking %s" % m[0]) 564 method = getattr(m[0],m[2]) 565 rv = None 566 try: 567 rv = method() 568 except: 569 self.logger.warn("Error from %s" % method, exc_info = True) 570 if not rv: 571 remove.append(m) 572 return remove
573 574 @locked
575 - def removeAll(self, remove):
576 """ 577 Finally, within another lock, call the "cleanup" 578 method on all the entries in remove, and remove 579 them from the official stuff list. (If stop_event 580 is set during execution, we return with the assumption 581 that Resources.cleanup() will take care of them) 582 """ 583 for r in remove: 584 if self.stop_event.isSet(): 585 return # Let cleanup handle this 586 self.logger.debug("Removing %s" % r[0]) 587 self.safeClean(r) 588 self.stuff.remove(r)
589 590 @locked
591 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
592 entry = (object,cleanupMethod,checkMethod) 593 self.logger.debug("Adding object %s" % object) 594 self.stuff.append(entry)
595 596 @locked
597 - def cleanup(self):
598 self.stop_event.set() 599 for m in self.stuff: 600 self.safeClean(m) 601 self.stuff = None 602 self.logger.debug("Cleanup done")
603
604 - def safeClean(self, m):
605 try: 606 self.logger.debug("Cleaning %s" % m[0]) 607 method = getattr(m[0],m[1]) 608 method() 609 except: 610 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
611
612 - def __del__(self):
613 self.cleanup()
614
615 -class Environment:
616 """ 617 Simple class for creating an executable environment 618 """ 619
620 - def __init__(self, *args):
621 """ 622 Takes an number of environment variable names which 623 should be copied to the target environment if present 624 in the current execution environment. 625 """ 626 if sys.platform == "win32": 627 # Prevents SocketException. See ticket:1518 628 self.env = os.environ.copy() 629 else: 630 self.env = {} 631 for arg in args: 632 if os.environ.has_key(arg): 633 self.env[arg] = os.environ[arg]
634 - def __call__(self):
635 """ 636 Returns the environment map when called. 637 """ 638 return self.env
639
640 - def set(self, key, value):
641 """ 642 Manually sets a value in the target environment. 643 """ 644 self.env[key] = value
645
646 - def append(self, key, addition):
647 """ 648 Manually adds a value to the environment string 649 """ 650 if self.env.has_key(key): 651 self.env[key] = os.pathsep.join([self.env[key], addition]) 652 else: 653 self.set(key, addition)
654
655 # 656 # Miscellaneious utilities 657 # 658 659 -def get_user_dir():
660 try: 661 from win32com.shell import shellcon, shell 662 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 663 except ImportError: 664 homeprop = os.path.expanduser("~") 665 return homeprop
666
667 -def edit_path(path_or_obj, start_text):
668 f = path.path(path_or_obj) 669 editor = os.getenv("VISUAL") or os.getenv("EDITOR") 670 if not editor: 671 if platform.system() == "Windows": 672 editor = "Notepad.exe" 673 else: 674 editor = "vi" 675 f.write_text(start_text) 676 from which import which 677 editor_path = which(editor) 678 pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f) 679 if pid: 680 re = RuntimeError("Couldn't spawn editor: %s" % editor) 681 re.pid = pid 682 raise re
683
684 #From: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/157035 685 -def tail_lines(filename,linesback=10,returnlist=0):
686 """Does what "tail -10 filename" would have done 687 Parameters:: 688 filename file to read 689 linesback Number of lines to read from end of file 690 returnlist Return a list containing the lines instead of a string 691 692 """ 693 avgcharsperline=75 694 695 file = open(filename,'r') 696 while 1: 697 try: file.seek(-1 * avgcharsperline * linesback,2) 698 except IOError: file.seek(0) 699 if file.tell() == 0: atstart=1 700 else: atstart=0 701 702 lines=file.read().split("\n") 703 if (len(lines) > (linesback+1)) or atstart: break 704 #The lines are bigger than we thought 705 avgcharsperline=avgcharsperline * 1.3 #Inc avg for retry 706 file.close() 707 708 if len(lines) > linesback: start=len(lines)-linesback -1 709 else: start=0 710 if returnlist: return lines[start:len(lines)-1] 711 712 out="" 713 for l in lines[start:len(lines)-1]: out=out + l + "\n" 714 return out
715