Package omero :: Package util
[hide private]
[frames] | no frames]

Source Code for Package omero.util

  1  #!/usr/bin/env python 
  2  # 
  3  # OMERO Utilities package 
  4  # 
  5  # Copyright 2009 Glencoe Software, Inc.  All Rights Reserved. 
  6  # Use is subject to license terms supplied in LICENSE.txt 
  7  # 
  8   
  9  import os 
 10  import sys 
 11  import Ice 
 12  import time 
 13  import uuid 
 14  import omero 
 15  import IceGrid 
 16  import logging 
 17  import Glacier2 
 18  import threading 
 19  import exceptions 
 20  import logging.handlers 
 21  import omero.util.concurrency 
 22   
 23  from omero.util.decorators import locked 
 24   
 25  LOGDIR = os.path.join("var","log") 
 26  LOGFORMAT =  """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s""" 
 27  LOGLEVEL = logging.INFO 
 28  LOGSIZE = 500000000 
 29  LOGNUM = 9 
 30  LOGMODE = "a" 
 31   
 32  orig_stdout = sys.stdout 
 33  orig_stderr = sys.stderr 
34 35 -def make_logname(self):
36 """ 37 Generates a logname from the given instance using the module and name from its class 38 """ 39 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__) 40 return log_name
41
42 -def configure_logging(logdir = None, logfile = None, loglevel = LOGLEVEL,\ 43 format = LOGFORMAT, filemode = LOGMODE, maxBytes = LOGSIZE, backupCount = LOGNUM, time_rollover = False):
44 45 if logdir is None or logfile is None: 46 handler = logging.StreamHandler() 47 elif not time_rollover: 48 handler = logging.handlers.RotatingFileHandler(os.path.join(logdir, logfile), maxBytes = maxBytes, backupCount = backupCount) 49 else: 50 handler = logging.handlers.TimedRotatingFileHandler(os.path.join(logdir, logfile),'midnight',1) 51 # Windows will not allow renaming (or deleting) a file that's open. 52 # There's nothing the logging package can do about that. 53 try: 54 sys.getwindowsversion() 55 except: 56 handler.doRollover() 57 58 handler.setLevel(loglevel) 59 formatter = logging.Formatter(format) 60 handler.setFormatter(formatter) 61 rootLogger = logging.getLogger() 62 rootLogger.setLevel(loglevel) 63 rootLogger.addHandler(handler) 64 return rootLogger
65
66 -def configure_server_logging(props):
67 """ 68 Takes an Ice.Properties instance and configures logging 69 """ 70 program_name = props.getProperty("Ice.Admin.ServerId") 71 # Using Ice.ProgramName on Windows failed 72 log_name = program_name+".log" 73 log_timed = props.getPropertyWithDefault("omero.logging.timedlog","False")[0] in ('T', 't') 74 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 75 log_size = int(props.getPropertyWithDefault("omero.logging.logsize",str(LOGSIZE))) 76 log_num = int(props.getPropertyWithDefault("omero.logging.lognum",str(LOGNUM))) 77 log_level = int(props.getPropertyWithDefault("omero.logging.level",str(LOGLEVEL))) 78 configure_logging(LOGDIR, log_name, loglevel=log_level, maxBytes=log_size, backupCount=log_num, time_rollover = log_timed) 79 80 sys.stdout = StreamRedirect(logging.getLogger("stdout")) 81 sys.stderr = StreamRedirect(logging.getLogger("stderr"))
82
83 -class StreamRedirect(object):
84 """ 85 Since all server components should exclusively using the logging module 86 any output to stdout or stderr is caught and logged at "WARN". This is 87 useful, especially in the case of Windows, where stdout/stderr is eaten. 88 """ 89
90 - def __init__(self, logger):
91 self.logger = logger 92 self.internal = logging.getLogger("StreamRedirect") 93 self.softspace = False
94
95 - def flush(self):
96 pass
97
98 - def write(self, msg):
99 msg = msg.strip() 100 if msg: 101 self.logger.warn(msg)
102
103 - def __getattr__(self, name):
104 self.internal.warn("No attribute: %s" % name)
105
106 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
107 """ 108 Try to return a ServiceFactory from the grid. 109 110 Try a number of times then give up and raise the 111 last exception returned. This method will only 112 work internally to the grid, i.e. behind the Glacier2 113 firewall. It is intended for internal servers to 114 be able to create sessions for accessing the database. :: 115 communicator := Ice.Communicator used to find the registry 116 user := Username which should have a session created 117 group := Group into which the session should be logged 118 retries := Number of session creation retries before throwing 119 interval := Seconds between retries 120 client_uuid := Uuid of the client which should be used 121 """ 122 log = logging.getLogger("omero.utils") 123 if stop_event == None: 124 stop_event = omero.util.concurrency.get_event() 125 126 tryCount = 0 127 excpt = None 128 query = communicator.stringToProxy("IceGrid/Query") 129 query = IceGrid.QueryPrx.checkedCast(query) 130 131 if client_uuid is None: 132 client_uuid = str(uuid.uuid4()) 133 134 while tryCount < retries: 135 if stop_event.isSet(): # Something is shutting down, exit. 136 return None 137 try: 138 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0] 139 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz) 140 sf = blitz.create(user, None, {"omero.client.uuid":client_uuid}) 141 # Group currently unused. 142 return omero.api.ServiceFactoryPrx.checkedCast(sf) 143 except Exception, e: 144 tryCount += 1 145 log.info("Failed to get session on attempt %s", str(tryCount)) 146 excpt = e 147 stop_event.wait(interval) 148 149 log.warn("Reason: %s", str(excpt)) 150 raise excpt
151
152 -def create_admin_session(communicator):
153 """ 154 """ 155 reg = communicator.stringToProxy("IceGrid/Registry") 156 reg = IceGrid.RegistryPrx.checkedCast(reg) 157 adm = reg.createAdminSession('null', '') 158 return adm
159
160 -def add_grid_object(communicator, obj):
161 """ 162 """ 163 sid = communicator.identityToString(obj.ice_getIdentity()) 164 adm = create_admin_session(communicator) 165 prx = adm.getAdmin() 166 try: 167 try: 168 prx.addObject(obj) 169 except IceGrid.ObjectExistsException: 170 prx.updateObject(obj) 171 finally: 172 adm.destroy()
173
174 -def long_to_path(id, root=""):
175 """ 176 Converts a long to a path such that for all directiories only 177 a 1000 files and a 1000 subdirectories will be returned. 178 179 This method duplicates the logic in 180 ome.io.nio.AbstractFileSystemService.java:getPath() 181 """ 182 suffix = "" 183 remaining = id 184 dirno = 0 185 186 if id is None or id == "": 187 raise exceptions.Exception("Expecting a not-null id.") 188 189 id = long(id) 190 191 if id < 0: 192 raise exceptions.Exception("Expecting a non-negative id.") 193 194 while (remaining > 999): 195 remaining /= 1000 196 197 if remaining > 0: 198 dirno = remaining % 1000 199 suffix = os.path.join("Dir-%03d" % dirno, suffix) 200 201 return os.path.join(root, "%s%s" %(suffix,id))
202
203 -class ServerContext(object):
204 """ 205 Context passed to all servants. 206 207 server_id, communicator, and stop_event will be 208 constructed by the top-level Server instance. 209 210 A context instance may also be configured to hold 211 on to an internal session (ServiceFactoryPrx) and 212 keep it alive. 213 214 This instance obeys the Resources API and calls 215 sf.keepAlive(None) on every check call, but does 216 nothing on cleanup. The sf instance must be manually 217 cleaned as the final operation of a servant. 218 219 (Note: cleanup of the server context indicates 220 server shutdown, so should be infrequent) 221 """ 222
223 - def __init__(self, server_id, communicator, stop_event):
224 self._lock = threading.RLock() 225 self.logger = logging.getLogger("omero.util.ServerContext") 226 self.server_id = server_id 227 self.communicator = communicator 228 self.stop_event = stop_event
229
230 - def newSession(self):
231 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event)
232
233 - def hasSession(self):
234 return hasattr(self, "session")
235 236 @locked
237 - def getSession(self, recreate = True):
238 """ 239 Returns the ServiceFactoryPrx configured for the context if 240 available. If the context was not configured for sessions, 241 an ApiUsageException will be thrown: servants should know 242 whether or not they were configured for sessions. 243 See Servant(..., needs_session = True) 244 245 Otherwise, if there is no ServiceFactoryPrx, an attempt will 246 be made to create one if recreate == True. If the value is None 247 or non can be recreated, an InternalException will be thrown. 248 249 TODO : currently no arguments are provided for re-creating these, 250 but also not in Servant.__init__ 251 """ 252 if not self.hasSession(): 253 raise omero.ApiUsageException("Not configured for server connection") 254 255 if self.session: 256 try: 257 self.session.keepAlive(None) 258 except exceptions.Exception, e: 259 self.logger.warn("Connection failure: %s" % e) 260 self.session = None 261 262 if self.session is None and recreate: 263 try: 264 self.newSession() 265 self.logger.info("Established connection: %s" % e) 266 except exceptions.Exception, e: 267 self.logger.warn("Failed to establish connection: %s" % e) 268 269 if self.session is None: 270 raise omero.InternalException("No connection to server") 271 272 return self.session
273
274 - def check(self):
275 """ 276 Calls getSession() but always returns True. This keeps the context 277 available in the resources for later uses, and tries to re-establish 278 a connection in case Blitz goes down. 279 """ 280 try: 281 self.getSession() 282 except: 283 pass 284 return True
285
286 - def cleanup(self):
287 """ 288 Does nothing. Context clean up must happen manually 289 since later activities may want to reuse it. Servants using 290 a server connection should cleanup the instance *after* Resources 291 is cleaned up 292 """ 293 pass
294
295 -class Server(Ice.Application):
296 """ 297 Basic server implementation which can be used for 298 implementing a standalone python server which can 299 be started from icegridnode. 300 301 The servant implementation MUST have a constructor 302 which takes a single ServerContext argument AND 303 have a cleanup() method 304 305 Logging is configured relative to the current directory 306 to be in var/log by default. 307 308 Usage:: 309 310 if __name__ == "__main__": 311 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services","")) 312 sys.exit(app.main(sys.argv)) 313 314 app.impl now points to an instance of ServicesI 315 316 """ 317
318 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
319 320 self.impl_class = impl_class 321 self.adapter_name = adapter_name 322 self.identity = identity 323 self.logdir = logdir 324 self.stop_event = omero.util.concurrency.get_event()
325
326 - def run(self,args):
327 328 from omero.rtypes import ObjectFactories as rFactories 329 from omero.columns import ObjectFactories as cFactories 330 331 props = self.communicator().getProperties() 332 configure_server_logging(props) 333 334 self.logger = logging.getLogger("omero.util.Server") 335 self.logger.info("*"*80) 336 self.logger.info("Starting") 337 338 self.shutdownOnInterrupt() 339 340 try: 341 342 self.objectfactory = omero.clients.ObjectFactory() 343 self.objectfactory.registerObjectFactory(self.communicator()) 344 for of in rFactories.values() + cFactories.values(): 345 of.register(self.communicator()) 346 347 try: 348 serverid = self.communicator().getProperties().getProperty("Ice.ServerId") 349 ctx = ServerContext(serverid, self.communicator(), self.stop_event) 350 self.impl = self.impl_class(ctx) 351 getattr(self.impl, "cleanup") # Required per docs 352 except: 353 self.logger.error("Failed initialization", exc_info=1) 354 sys.exit(100) 355 356 try: 357 self.adapter = self.communicator().createObjectAdapter(self.adapter_name) 358 prx = self.adapter.add(self.impl, self.identity) 359 self.adapter.activate() 360 add_grid_object(self.communicator(), prx) # This must happen _after_ activation 361 except: 362 self.logger.error("Failed activation", exc_info=1) 363 sys.exit(200) 364 365 self.logger.info("Entering main loop") 366 self.communicator().waitForShutdown() 367 finally: 368 self.stop_event.set() # Let's all waits shutdown 369 self.logger.info("Cleanup") 370 self.cleanup() 371 self.logger.info("Stopped") 372 self.logger.info("*"*80)
373
374 - def cleanup(self):
375 """ 376 Cleans up all resources that were created by this server. 377 Primarily the one servant instance. 378 """ 379 if hasattr(self,"impl"): 380 try: 381 self.impl.cleanup() 382 finally: 383 del self.impl
384
385 386 -class SimpleServant(object):
387 """ 388 Base servant initialization. Doesn't create or try to cleanup 389 a top-level Resources thread. This is useful for large numbers 390 of servants. For servers and other singleton-like servants, 391 see "Servant" 392 """
393 - def __init__(self, ctx):
394 self._lock = threading.RLock() 395 self.ctx = ctx 396 self.stop_event = ctx.stop_event 397 self.communicator = ctx.communicator 398 self.logger = logging.getLogger(make_logname(self)) 399 self.logger.debug("Created")
400
401 -class Servant(SimpleServant):
402 """ 403 Abstract servant which can be used along with a slice2py 404 generated dispatch class as the base type of high-level servants. 405 These provide resource cleanup as per the omero.util.Server 406 class. 407 408 By passing "needs_session = True" to this constructor, an internal 409 session will be created and stored in ServerContext as well as 410 registered with self.resources 411 """ 412
413 - def __init__(self, ctx, needs_session = False):
414 SimpleServant.__init__(self, ctx) 415 self.resources = omero.util.Resources(sleeptime = 60, stop_event = self.stop_event) 416 if needs_session: 417 self.ctx.newSession() 418 self.resources.add(self.ctx)
419
420 - def cleanup(self):
421 """ 422 Cleanups all resoures created by this servant. Calling 423 cleanup multiple times should be safe. 424 """ 425 resources = self.resources 426 self.resources = None 427 if resources != None: 428 self.logger.info("Cleaning up") 429 resources.cleanup() 430 self.logger.info("Done") 431 if self.ctx.hasSession(): 432 try: 433 sf = self.ctx.getSession(recreate=False) 434 self.logger.debug("Destroying %s" % sf) 435 sf.destroy() 436 except: 437 pass
438
439 - def __del__(self):
440 self.cleanup()
441
442 443 -class Resources:
444 """ 445 Container class for storing resources which should be 446 cleaned up on close and periodically checked. Use 447 stop_event.set() to stop the internal thread. 448 """ 449
450 - def __init__(self, sleeptime = 60, stop_event = None):
451 """ 452 Add resources via add(object). They should have a no-arg cleanup() 453 and a check() method. 454 455 The check method will be called periodically (default: 60 seconds) 456 on each resource. The cleanup method will be called on 457 Resources.cleanup() 458 """ 459 460 self._lock = threading.RLock() 461 self.logger = logging.getLogger("omero.util.Resources") 462 self.stop_event = stop_event 463 if not self.stop_event: 464 self.stop_event = omero.util.concurrency.get_event() 465 466 if sleeptime < 5: 467 raise exceptions.Exception("Sleep time should be greater than 5: " % sleeptime) 468 469 self.sleeptime = sleeptime 470 self.stuff = [] 471 472 class Task(threading.Thread): 473 """ 474 Internal thread used for checking "stuff" 475 """ 476 def run(self): 477 ctx = self.ctx # Outer class 478 ctx.logger.info("Starting") 479 while not ctx.stop_event.isSet(): 480 try: 481 ctx.logger.debug("Executing") 482 copy = ctx.copyStuff() 483 remove = ctx.checkAll(copy) 484 ctx.removeAll(remove) 485 except: 486 ctx.logger.error("Exception during execution", exc_info = True) 487 488 ctx.logger.debug("Sleeping %s" % ctx.sleeptime) 489 # ticket:1531 - Attempting to catch threading issues 490 try: 491 ctx.stop_event.wait(ctx.sleeptime) 492 except ValueError: 493 pass 494 495 ctx.logger.info("Halted")
496 497 self.thread = Task() 498 self.thread.ctx = self 499 self.thread.start() 500 501 @locked
502 - def copyStuff(self):
503 """ 504 Within a lock, copy the "stuff" list and reverse it. 505 The list is reversed so that entries added 506 later, which may depend on earlier added entries 507 get a chance to be cleaned up first. 508 """ 509 copy = list(self.stuff) 510 copy.reverse() 511 return copy
512 513 # Not locked
514 - def checkAll(self, copy):
515 """ 516 While stop_event is unset, go through the copy 517 of stuff and call the check method on each 518 entry. Any that throws an exception or returns 519 a False value will be returned in the remove list. 520 """ 521 remove = [] 522 for m in copy: 523 if self.stop_event.isSet(): 524 return # Let cleanup handle this 525 self.logger.debug("Checking %s" % m[0]) 526 method = getattr(m[0],m[2]) 527 rv = None 528 try: 529 rv = method() 530 except: 531 self.logger.warn("Error from %s" % method, exc_info = True) 532 if not rv: 533 remove.append(m) 534 return remove
535 536 @locked
537 - def removeAll(self, remove):
538 """ 539 Finally, within another lock, call the "cleanup" 540 method on all the entries in remove, and remove 541 them from the official stuff list. (If stop_event 542 is set during execution, we return with the assumption 543 that Resources.cleanup() will take care of them) 544 """ 545 for r in remove: 546 if self.stop_event.isSet(): 547 return # Let cleanup handle this 548 self.logger.debug("Removing %s" % r[0]) 549 self.safeClean(r) 550 self.stuff.remove(r)
551 552 @locked
553 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
554 entry = (object,cleanupMethod,checkMethod) 555 self.logger.info("Adding object %s" % object) 556 self.stuff.append(entry)
557 558 @locked
559 - def cleanup(self):
560 self.stop_event.set() 561 for m in self.stuff: 562 self.safeClean(m) 563 self.stuff = None 564 self.logger.info("Cleanup done")
565
566 - def safeClean(self, m):
567 try: 568 self.logger.debug("Cleaning %s" % m[0]) 569 method = getattr(m[0],m[1]) 570 method() 571 except: 572 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
573
574 - def __del__(self):
575 self.cleanup()
576
577 -class Environment:
578 """ 579 Simple class for creating an executable environment 580 """ 581
582 - def __init__(self, *args):
583 """ 584 Takes an number of environment variable names which 585 should be copied to the target environment if present 586 in the current execution environment. 587 """ 588 if sys.platform == "win32": 589 # Prevents SocketException. See ticket:1518 590 self.env = os.environ.copy() 591 else: 592 self.env = {} 593 for arg in args: 594 if os.environ.has_key(arg): 595 self.env[arg] = os.environ[arg]
596 - def __call__(self):
597 """ 598 Returns the environment map when called. 599 """ 600 return self.env
601
602 - def set(self, key, value):
603 """ 604 Manually sets a value in the target environment. 605 """ 606 self.env[key] = value
607
608 - def append(self, key, addition):
609 """ 610 Manually adds a value to the environment string 611 """ 612 if self.env.has_key(key): 613 self.env[key] = os.pathsep.join([self.env[key], addition]) 614 else: 615 self.set(key, addition)
616