Package omero :: Package util
[hide private]
[frames] | no frames]

Source Code for Package omero.util

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Utilities package 
  4  # 
  5  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  6  # Use is subject to license terms supplied in LICENSE.txt 
  7  # 
  8   
  9  import os 
 10  import sys 
 11  import Ice 
 12  import path 
 13  import time 
 14  import uuid 
 15  import omero 
 16  import IcePy 
 17  import IceGrid 
 18  import logging 
 19  import platform 
 20  import Glacier2 
 21  import threading 
 22  import exceptions 
 23  import logging.handlers 
 24  import omero.util.concurrency 
 25   
 26  from omero.util.decorators import locked 
 27   
 28  LOGDIR = os.path.join("var","log") 
 29  LOGFORMAT =  """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s""" 
 30  LOGLEVEL = logging.INFO 
 31  LOGSIZE = 500000000 
 32  LOGNUM = 9 
 33  LOGMODE = "a" 
 34   
 35  orig_stdout = sys.stdout 
 36  orig_stderr = sys.stderr 
37 38 -def make_logname(self):
39 """ 40 Generates a logname from the given instance using the module and name from its class 41 """ 42 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__) 43 return log_name
44
45 -def configure_logging(logdir = None, logfile = None, loglevel = LOGLEVEL,\ 46 format = LOGFORMAT, filemode = LOGMODE, maxBytes = LOGSIZE, backupCount = LOGNUM, time_rollover = False):
47 48 if logdir is None or logfile is None: 49 handler = logging.StreamHandler() 50 elif not time_rollover: 51 handler = logging.handlers.RotatingFileHandler(os.path.join(logdir, logfile), maxBytes = maxBytes, backupCount = backupCount) 52 else: 53 handler = logging.handlers.TimedRotatingFileHandler(os.path.join(logdir, logfile),'midnight',1) 54 # Windows will not allow renaming (or deleting) a file that's open. 55 # There's nothing the logging package can do about that. 56 try: 57 sys.getwindowsversion() 58 except: 59 handler.doRollover() 60 61 handler.setLevel(loglevel) 62 formatter = logging.Formatter(format) 63 handler.setFormatter(formatter) 64 rootLogger = logging.getLogger() 65 rootLogger.setLevel(loglevel) 66 rootLogger.addHandler(handler) 67 return rootLogger
68
69 -def configure_server_logging(props):
70 """ 71 Takes an Ice.Properties instance and configures logging 72 """ 73 program_name = props.getProperty("Ice.Admin.ServerId") 74 # Using Ice.ProgramName on Windows failed 75 log_name = program_name+".log" 76 log_timed = props.getPropertyWithDefault("omero.logging.timedlog","False")[0] in ('T', 't') 77 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 78 log_size = int(props.getPropertyWithDefault("omero.logging.logsize",str(LOGSIZE))) 79 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 80 log_level = int(props.getPropertyWithDefault("omero.logging.level",str(LOGLEVEL))) 81 configure_logging(LOGDIR, log_name, loglevel=log_level, maxBytes=log_size, backupCount=log_num, time_rollover = log_timed) 82 83 sys.stdout = StreamRedirect(logging.getLogger("stdout")) 84 sys.stderr = StreamRedirect(logging.getLogger("stderr"))
85
86 -class StreamRedirect(object):
87 """ 88 Since all server components should exclusively using the logging module 89 any output to stdout or stderr is caught and logged at "WARN". This is 90 useful, especially in the case of Windows, where stdout/stderr is eaten. 91 """ 92
93 - def __init__(self, logger):
94 self.logger = logger 95 self.internal = logging.getLogger("StreamRedirect") 96 self.softspace = False
97
98 - def flush(self):
99 pass
100
101 - def write(self, msg):
102 msg = msg.strip() 103 if msg: 104 self.logger.warn(msg)
105
106 - def __getattr__(self, name):
107 self.internal.warn("No attribute: %s" % name)
108
109 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
110 """ 111 Try to return a ServiceFactory from the grid. 112 113 Try a number of times then give up and raise the 114 last exception returned. This method will only 115 work internally to the grid, i.e. behind the Glacier2 116 firewall. It is intended for internal servers to 117 be able to create sessions for accessing the database. :: 118 communicator := Ice.Communicator used to find the registry 119 user := Username which should have a session created 120 group := Group into which the session should be logged 121 retries := Number of session creation retries before throwing 122 interval := Seconds between retries 123 client_uuid := Uuid of the client which should be used 124 """ 125 log = logging.getLogger("omero.utils") 126 if stop_event == None: 127 stop_event = omero.util.concurrency.get_event(name="internal_service_factory") 128 129 tryCount = 0 130 excpt = None 131 query = communicator.stringToProxy("IceGrid/Query") 132 query = IceGrid.QueryPrx.checkedCast(query) 133 134 import omero_Constants_ice 135 implicit_ctx = communicator.getImplicitContext() 136 if client_uuid is not None: 137 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid) 138 else: 139 if not implicit_ctx.containsKey(omero.constants.CLIENTUUID): 140 client_uuid = str(uuid.uuid4()) 141 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid) 142 143 while tryCount < retries: 144 if stop_event.isSet(): # Something is shutting down, exit. 145 return None 146 try: 147 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0] 148 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz) 149 sf = blitz.create(user, None) 150 # Group currently unused. 151 return omero.api.ServiceFactoryPrx.checkedCast(sf) 152 except Exception, e: 153 tryCount += 1 154 log.info("Failed to get session on attempt %s", str(tryCount)) 155 excpt = e 156 stop_event.wait(interval) 157 158 log.warn("Reason: %s", str(excpt)) 159 raise excpt
160
161 -def create_admin_session(communicator):
162 """ 163 """ 164 reg = communicator.stringToProxy("IceGrid/Registry") 165 reg = IceGrid.RegistryPrx.checkedCast(reg) 166 adm = reg.createAdminSession('null', '') 167 return adm
168
169 -def add_grid_object(communicator, obj):
170 """ 171 """ 172 sid = communicator.identityToString(obj.ice_getIdentity()) 173 adm = create_admin_session(communicator) 174 prx = adm.getAdmin() 175 try: 176 try: 177 prx.addObject(obj) 178 except IceGrid.ObjectExistsException: 179 prx.updateObject(obj) 180 finally: 181 adm.destroy()
182
183 -def long_to_path(id, root=""):
184 """ 185 Converts a long to a path such that for all directiories only 186 a 1000 files and a 1000 subdirectories will be returned. 187 188 This method duplicates the logic in 189 ome.io.nio.AbstractFileSystemService.java:getPath() 190 """ 191 suffix = "" 192 remaining = id 193 dirno = 0 194 195 if id is None or id == "": 196 raise exceptions.Exception("Expecting a not-null id.") 197 198 id = long(id) 199 200 if id < 0: 201 raise exceptions.Exception("Expecting a non-negative id.") 202 203 while (remaining > 999): 204 remaining /= 1000 205 206 if remaining > 0: 207 dirno = remaining % 1000 208 suffix = os.path.join("Dir-%03d" % dirno, suffix) 209 210 return os.path.join(root, "%s%s" %(suffix,id))
211
212 -class ServerContext(object):
213 """ 214 Context passed to all servants. 215 216 server_id, communicator, and stop_event will be 217 constructed by the top-level Server instance. 218 219 A context instance may also be configured to hold 220 on to an internal session (ServiceFactoryPrx) and 221 keep it alive. 222 223 This instance obeys the Resources API and calls 224 sf.keepAlive(None) on every check call, but does 225 nothing on cleanup. The sf instance must be manually 226 cleaned as the final operation of a servant. 227 228 (Note: cleanup of the server context indicates 229 server shutdown, so should be infrequent) 230 """ 231
232 - def __init__(self, server_id, communicator, stop_event, on_newsession = None):
233 self._lock = threading.RLock() 234 self.logger = logging.getLogger("omero.util.ServerContext") 235 self.server_id = server_id 236 self.communicator = communicator 237 self.stop_event = stop_event 238 self.servant_map = dict() 239 self.on_newsession = None
240 241 @locked
242 - def add_servant(self, adapter_or_current, servant, ice_identity = None):
243 oa = adapter_or_current 244 if isinstance(adapter_or_current, (Ice.Current, IcePy.Current)): 245 oa = oa.adapter 246 if ice_identity is None: 247 prx = oa.addWithUUID(servant) 248 else: 249 prx = oa.add(servant, ice_identity) 250 251 servant.setProxy(prx) 252 self.servant_map[prx] = servant 253 return prx
254
255 - def newSession(self):
256 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event) 257 if callable(self.on_newsession): 258 self.on_newsession(self.session)
259
260 - def hasSession(self):
261 return hasattr(self, "session")
262 263 @locked
264 - def getSession(self, recreate = True):
265 """ 266 Returns the ServiceFactoryPrx configured for the context if 267 available. If the context was not configured for sessions, 268 an ApiUsageException will be thrown: servants should know 269 whether or not they were configured for sessions. 270 See Servant(..., needs_session = True) 271 272 Otherwise, if there is no ServiceFactoryPrx, an attempt will 273 be made to create one if recreate == True. If the value is None 274 or non can be recreated, an InternalException will be thrown. 275 276 TODO : currently no arguments are provided for re-creating these, 277 but also not in Servant.__init__ 278 """ 279 if not self.hasSession(): 280 raise omero.ApiUsageException("Not configured for server connection") 281 282 if self.session: 283 try: 284 self.session.keepAlive(None) 285 except Ice.CommunicatorDestroyedException: 286 self.session = None # Ignore 287 except exceptions.Exception, e: 288 self.logger.warn("Connection failure: %s" % e) 289 self.session = None 290 291 if self.session is None and recreate: 292 try: 293 self.newSession() 294 self.logger.info("Established connection: %s" % self.session) 295 except exceptions.Exception, e: 296 self.logger.warn("Failed to establish connection: %s" % e) 297 298 if self.session is None: 299 raise omero.InternalException("No connection to server") 300 301 return self.session
302
303 - def check(self):
304 """ 305 Calls getSession() but always returns True. This keeps the context 306 available in the resources for later uses, and tries to re-establish 307 a connection in case Blitz goes down. 308 """ 309 try: 310 self.getSession() 311 except: 312 pass 313 return True
314
315 - def cleanup(self):
316 """ 317 Does nothing. Context clean up must happen manually 318 since later activities may want to reuse it. Servants using 319 a server connection should cleanup the instance *after* Resources 320 is cleaned up 321 """ 322 pass
323
324 -class Server(Ice.Application):
325 """ 326 Basic server implementation which can be used for 327 implementing a standalone python server which can 328 be started from icegridnode. 329 330 The servant implementation MUST have a constructor 331 which takes a single ServerContext argument AND 332 have a cleanup() method 333 334 Logging is configured relative to the current directory 335 to be in var/log by default. 336 337 Usage:: 338 339 if __name__ == "__main__": 340 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services","")) 341 sys.exit(app.main(sys.argv)) 342 343 app.impl now points to an instance of ServicesI 344 345 """ 346
347 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
348 349 self.impl_class = impl_class 350 self.adapter_name = adapter_name 351 self.identity = identity 352 self.logdir = logdir 353 self.stop_event = omero.util.concurrency.get_event(name="Server")
354
355 - def run(self,args):
356 357 from omero.rtypes import ObjectFactories as rFactories 358 from omero.columns import ObjectFactories as cFactories 359 360 props = self.communicator().getProperties() 361 configure_server_logging(props) 362 363 self.logger = logging.getLogger("omero.util.Server") 364 self.logger.info("*"*80) 365 self.logger.info("Starting") 366 367 self.shutdownOnInterrupt() 368 369 try: 370 371 self.objectfactory = omero.clients.ObjectFactory() 372 self.objectfactory.registerObjectFactory(self.communicator()) 373 for of in rFactories.values() + cFactories.values(): 374 of.register(self.communicator()) 375 376 try: 377 serverid = self.communicator().getProperties().getProperty("Ice.ServerId") 378 ctx = ServerContext(serverid, self.communicator(), self.stop_event) 379 self.impl = self.impl_class(ctx) 380 getattr(self.impl, "cleanup") # Required per docs 381 except: 382 self.logger.error("Failed initialization", exc_info=1) 383 sys.exit(100) 384 385 try: 386 self.adapter = self.communicator().createObjectAdapter(self.adapter_name) 387 self.adapter.activate() 388 ctx.add_servant(self.adapter, self.impl, self.identity) # calls setProxy 389 add_grid_object(self.communicator(), self.impl.prx) # This must happen _after_ activation 390 except: 391 self.logger.error("Failed activation", exc_info=1) 392 sys.exit(200) 393 394 self.logger.info("Entering main loop") 395 self.communicator().waitForShutdown() 396 finally: 397 self.stop_event.set() # Let's all waits shutdown 398 self.logger.info("Cleanup") 399 self.cleanup() 400 self.logger.info("Stopped") 401 self.logger.info("*"*80)
402
403 - def cleanup(self):
404 """ 405 Cleans up all resources that were created by this server. 406 Primarily the one servant instance. 407 """ 408 if hasattr(self,"impl"): 409 try: 410 self.impl.cleanup() 411 finally: 412 del self.impl
413
414 415 -class SimpleServant(object):
416 """ 417 Base servant initialization. Doesn't create or try to cleanup 418 a top-level Resources thread. This is useful for large numbers 419 of servants. For servers and other singleton-like servants, 420 see "Servant" 421 """
422 - def __init__(self, ctx):
423 self._lock = threading.RLock() 424 self.prx = None # Proxy which points to self 425 self.ctx = ctx 426 self.stop_event = ctx.stop_event 427 self.communicator = ctx.communicator 428 self.logger = logging.getLogger(make_logname(self)) 429 self.logger.debug("Created")
430
431 - def setProxy(self, prx):
432 """ 433 Should be overwritten for post-initialization activities. 434 The reason this method exists is that the implementation 435 must be complete before registering it with the adapter. 436 """ 437 self.prx = prx
438
439 -class Servant(SimpleServant):
440 """ 441 Abstract servant which can be used along with a slice2py 442 generated dispatch class as the base type of high-level servants. 443 These provide resource cleanup as per the omero.util.Server 444 class. 445 446 By passing "needs_session = True" to this constructor, an internal 447 session will be created and stored in ServerContext as well as 448 registered with self.resources 449 """ 450
451 - def __init__(self, ctx, needs_session = False):
452 SimpleServant.__init__(self, ctx) 453 self.resources = omero.util.Resources(sleeptime = 60, stop_event = self.stop_event) 454 if needs_session: 455 self.ctx.newSession() 456 self.resources.add(self.ctx)
457
458 - def cleanup(self):
459 """ 460 Cleanups all resoures created by this servant. Calling 461 cleanup multiple times should be safe. 462 """ 463 resources = self.resources 464 self.resources = None 465 if resources != None: 466 self.logger.info("Cleaning up") 467 resources.cleanup() 468 self.logger.info("Done") 469 if self.ctx.hasSession(): 470 try: 471 sf = self.ctx.getSession(recreate=False) 472 self.logger.debug("Destroying %s" % sf) 473 sf.destroy() 474 except: 475 pass
476
477 - def __del__(self):
478 self.cleanup()
479
480 481 -class Resources:
482 """ 483 Container class for storing resources which should be 484 cleaned up on close and periodically checked. Use 485 stop_event.set() to stop the internal thread. 486 """ 487
488 - def __init__(self, sleeptime = 60, stop_event = None):
489 """ 490 Add resources via add(object). They should have a no-arg cleanup() 491 and a check() method. 492 493 The check method will be called periodically (default: 60 seconds) 494 on each resource. The cleanup method will be called on 495 Resources.cleanup() 496 """ 497 498 self.stuff = [] 499 self._lock = threading.RLock() 500 self.logger = logging.getLogger("omero.util.Resources") 501 self.stop_event = stop_event 502 if not self.stop_event: 503 self.stop_event = omero.util.concurrency.get_event(name="Resources") 504 505 if sleeptime < 5: 506 raise exceptions.Exception("Sleep time should be greater than 5: %s" % sleeptime) 507 508 self.sleeptime = sleeptime 509 510 class Task(threading.Thread): 511 """ 512 Internal thread used for checking "stuff" 513 """ 514 def run(self): 515 ctx = self.ctx # Outer class 516 ctx.logger.info("Starting") 517 while not ctx.stop_event.isSet(): 518 try: 519 ctx.logger.debug("Executing") 520 copy = ctx.copyStuff() 521 remove = ctx.checkAll(copy) 522 ctx.removeAll(remove) 523 except: 524 ctx.logger.error("Exception during execution", exc_info = True) 525 526 ctx.logger.debug("Sleeping %s" % ctx.sleeptime) 527 # ticket:1531 - Attempting to catch threading issues 528 try: 529 ctx.stop_event.wait(ctx.sleeptime) 530 except ValueError: 531 pass 532 533 if isinstance(ctx.stop_event, omero.util.concurrency.AtExitEvent): 534 if ctx.stop_event.atexit: 535 return # Skipping log. See #3260 536 537 ctx.logger.info("Halted")
538 539 self.thread = Task() 540 self.thread.ctx = self 541 self.thread.start() 542 543 @locked
544 - def copyStuff(self):
545 """ 546 Within a lock, copy the "stuff" list and reverse it. 547 The list is reversed so that entries added 548 later, which may depend on earlier added entries 549 get a chance to be cleaned up first. 550 """ 551 copy = list(self.stuff) 552 copy.reverse() 553 return copy
554 555 # Not locked
556 - def checkAll(self, copy):
557 """ 558 While stop_event is unset, go through the copy 559 of stuff and call the check method on each 560 entry. Any that throws an exception or returns 561 a False value will be returned in the remove list. 562 """ 563 remove = [] 564 for m in copy: 565 if self.stop_event.isSet(): 566 return # Let cleanup handle this 567 self.logger.debug("Checking %s" % m[0]) 568 method = getattr(m[0],m[2]) 569 rv = None 570 try: 571 rv = method() 572 except: 573 self.logger.warn("Error from %s" % method, exc_info = True) 574 if not rv: 575 remove.append(m) 576 return remove
577 578 @locked
579 - def removeAll(self, remove):
580 """ 581 Finally, within another lock, call the "cleanup" 582 method on all the entries in remove, and remove 583 them from the official stuff list. (If stop_event 584 is set during execution, we return with the assumption 585 that Resources.cleanup() will take care of them) 586 """ 587 for r in remove: 588 if self.stop_event.isSet(): 589 return # Let cleanup handle this 590 self.logger.debug("Removing %s" % r[0]) 591 self.safeClean(r) 592 self.stuff.remove(r)
593 594 @locked
595 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
596 entry = (object,cleanupMethod,checkMethod) 597 self.logger.debug("Adding object %s" % object) 598 self.stuff.append(entry)
599 600 @locked
601 - def cleanup(self):
602 self.stop_event.set() 603 for m in self.stuff: 604 self.safeClean(m) 605 self.stuff = None 606 self.logger.debug("Cleanup done")
607
608 - def safeClean(self, m):
609 try: 610 self.logger.debug("Cleaning %s" % m[0]) 611 method = getattr(m[0],m[1]) 612 method() 613 except: 614 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
615
616 - def __del__(self):
617 self.cleanup()
618
619 -class Environment:
620 """ 621 Simple class for creating an executable environment 622 """ 623
624 - def __init__(self, *args):
625 """ 626 Takes an number of environment variable names which 627 should be copied to the target environment if present 628 in the current execution environment. 629 """ 630 if sys.platform == "win32": 631 # Prevents SocketException. See ticket:1518 632 self.env = os.environ.copy() 633 else: 634 self.env = {} 635 for arg in args: 636 if os.environ.has_key(arg): 637 self.env[arg] = os.environ[arg]
638 - def __call__(self):
639 """ 640 Returns the environment map when called. 641 """ 642 return self.env
643
644 - def set(self, key, value):
645 """ 646 Manually sets a value in the target environment. 647 """ 648 self.env[key] = value
649
650 - def append(self, key, addition):
651 """ 652 Manually adds a value to the environment string 653 """ 654 if self.env.has_key(key): 655 self.env[key] = os.pathsep.join([self.env[key], addition]) 656 else: 657 self.set(key, addition)
658
659 # 660 # Miscellaneious utilities 661 # 662 663 -def get_user_dir():
664 exceptions_to_handle = (ImportError) 665 try: 666 from pywintypes import com_error 667 from win32com.shell import shellcon, shell 668 exceptions_to_handle = (ImportError, com_error) 669 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 670 except exceptions_to_handle: 671 homeprop = os.path.expanduser("~") 672 return homeprop
673
674 -def edit_path(path_or_obj, start_text):
675 f = path.path(path_or_obj) 676 editor = os.getenv("VISUAL") or os.getenv("EDITOR") 677 if not editor: 678 if platform.system() == "Windows": 679 editor = "Notepad.exe" 680 else: 681 editor = "vi" 682 f.write_text(start_text) 683 from which import which 684 editor_path = which(editor) 685 pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f) 686 if pid: 687 re = RuntimeError("Couldn't spawn editor: %s" % editor) 688 re.pid = pid 689 raise re
690
691 #From: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/157035 692 -def tail_lines(filename,linesback=10,returnlist=0):
693 """Does what "tail -10 filename" would have done 694 Parameters:: 695 filename file to read 696 linesback Number of lines to read from end of file 697 returnlist Return a list containing the lines instead of a string 698 699 """ 700 avgcharsperline=75 701 702 file = open(filename,'r') 703 while 1: 704 try: file.seek(-1 * avgcharsperline * linesback,2) 705 except IOError: file.seek(0) 706 if file.tell() == 0: atstart=1 707 else: atstart=0 708 709 lines=file.read().split("\n") 710 if (len(lines) > (linesback+1)) or atstart: break 711 #The lines are bigger than we thought 712 avgcharsperline=avgcharsperline * 1.3 #Inc avg for retry 713 file.close() 714 715 if len(lines) > linesback: start=len(lines)-linesback -1 716 else: start=0 717 if returnlist: return lines[start:len(lines)-1] 718 719 out="" 720 for l in lines[start:len(lines)-1]: out=out + l + "\n" 721 return out
722