1
2
3
4
5
6
7
8
9 import os
10 import sys
11 import Ice
12 import time
13 import uuid
14 import omero
15 import IceGrid
16 import logging
17 import Glacier2
18 import threading
19 import exceptions
20 import logging.handlers
21 import omero.util.concurrency
22
23 from omero.util.decorators import locked
24
25 LOGDIR = os.path.join("var","log")
26 LOGFORMAT = """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s"""
27 LOGLEVEL = logging.INFO
28 LOGSIZE = 500000000
29 LOGNUM = 9
30 LOGMODE = "a"
31
32 orig_stdout = sys.stdout
33 orig_stderr = sys.stderr
36 """
37 Generates a logname from the given instance using the module and name from its class
38 """
39 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
40 return log_name
41
65
82
84 """
85 Since all server components should exclusively using the logging module
86 any output to stdout or stderr is caught and logged at "WARN". This is
87 useful, especially in the case of Windows, where stdout/stderr is eaten.
88 """
89
91 self.logger = logger
92 self.internal = logging.getLogger("StreamRedirect")
93 self.softspace = False
94
97
99 msg = msg.strip()
100 if msg:
101 self.logger.warn(msg)
102
104 self.internal.warn("No attribute: %s" % name)
105
106 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
107 """
108 Try to return a ServiceFactory from the grid.
109
110 Try a number of times then give up and raise the
111 last exception returned. This method will only
112 work internally to the grid, i.e. behind the Glacier2
113 firewall. It is intended for internal servers to
114 be able to create sessions for accessing the database. ::
115 communicator := Ice.Communicator used to find the registry
116 user := Username which should have a session created
117 group := Group into which the session should be logged
118 retries := Number of session creation retries before throwing
119 interval := Seconds between retries
120 client_uuid := Uuid of the client which should be used
121 """
122 log = logging.getLogger("omero.utils")
123 if stop_event == None:
124 stop_event = omero.util.concurrency.get_event()
125
126 tryCount = 0
127 excpt = None
128 query = communicator.stringToProxy("IceGrid/Query")
129 query = IceGrid.QueryPrx.checkedCast(query)
130
131 if client_uuid is None:
132 client_uuid = str(uuid.uuid4())
133
134 while tryCount < retries:
135 if stop_event.isSet():
136 return None
137 try:
138 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0]
139 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz)
140 sf = blitz.create(user, None, {"omero.client.uuid":client_uuid})
141
142 return omero.api.ServiceFactoryPrx.checkedCast(sf)
143 except Exception, e:
144 tryCount += 1
145 log.info("Failed to get session on attempt %s", str(tryCount))
146 excpt = e
147 stop_event.wait(interval)
148
149 log.warn("Reason: %s", str(excpt))
150 raise excpt
151
153 """
154 """
155 reg = communicator.stringToProxy("IceGrid/Registry")
156 reg = IceGrid.RegistryPrx.checkedCast(reg)
157 adm = reg.createAdminSession('null', '')
158 return adm
159
161 """
162 """
163 sid = communicator.identityToString(obj.ice_getIdentity())
164 adm = create_admin_session(communicator)
165 prx = adm.getAdmin()
166 try:
167 try:
168 prx.addObject(obj)
169 except IceGrid.ObjectExistsException:
170 prx.updateObject(obj)
171 finally:
172 adm.destroy()
173
175 """
176 Converts a long to a path such that for all directiories only
177 a 1000 files and a 1000 subdirectories will be returned.
178
179 This method duplicates the logic in
180 ome.io.nio.AbstractFileSystemService.java:getPath()
181 """
182 suffix = ""
183 remaining = id
184 dirno = 0
185
186 if id is None or id == "":
187 raise exceptions.Exception("Expecting a not-null id.")
188
189 id = long(id)
190
191 if id < 0:
192 raise exceptions.Exception("Expecting a non-negative id.")
193
194 while (remaining > 999):
195 remaining /= 1000
196
197 if remaining > 0:
198 dirno = remaining % 1000
199 suffix = os.path.join("Dir-%03d" % dirno, suffix)
200
201 return os.path.join(root, "%s%s" %(suffix,id))
202
203 -class ServerContext(object):
204 """
205 Context passed to all servants.
206
207 server_id, communicator, and stop_event will be
208 constructed by the top-level Server instance.
209
210 A context instance may also be configured to hold
211 on to an internal session (ServiceFactoryPrx) and
212 keep it alive.
213
214 This instance obeys the Resources API and calls
215 sf.keepAlive(None) on every check call, but does
216 nothing on cleanup. The sf instance must be manually
217 cleaned as the final operation of a servant.
218
219 (Note: cleanup of the server context indicates
220 server shutdown, so should be infrequent)
221 """
222
223 - def __init__(self, server_id, communicator, stop_event):
224 self._lock = threading.RLock()
225 self.logger = logging.getLogger("omero.util.ServerContext")
226 self.server_id = server_id
227 self.communicator = communicator
228 self.stop_event = stop_event
229
230 - def newSession(self):
231 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event)
232
233 - def hasSession(self):
234 return hasattr(self, "session")
235
236 @locked
237 - def getSession(self, recreate = True):
238 """
239 Returns the ServiceFactoryPrx configured for the context if
240 available. If the context was not configured for sessions,
241 an ApiUsageException will be thrown: servants should know
242 whether or not they were configured for sessions.
243 See Servant(..., needs_session = True)
244
245 Otherwise, if there is no ServiceFactoryPrx, an attempt will
246 be made to create one if recreate == True. If the value is None
247 or non can be recreated, an InternalException will be thrown.
248
249 TODO : currently no arguments are provided for re-creating these,
250 but also not in Servant.__init__
251 """
252 if not self.hasSession():
253 raise omero.ApiUsageException("Not configured for server connection")
254
255 if self.session:
256 try:
257 self.session.keepAlive(None)
258 except exceptions.Exception, e:
259 self.logger.warn("Connection failure: %s" % e)
260 self.session = None
261
262 if self.session is None and recreate:
263 try:
264 self.newSession()
265 self.logger.info("Established connection: %s" % e)
266 except exceptions.Exception, e:
267 self.logger.warn("Failed to establish connection: %s" % e)
268
269 if self.session is None:
270 raise omero.InternalException("No connection to server")
271
272 return self.session
273
275 """
276 Calls getSession() but always returns True. This keeps the context
277 available in the resources for later uses, and tries to re-establish
278 a connection in case Blitz goes down.
279 """
280 try:
281 self.getSession()
282 except:
283 pass
284 return True
285
287 """
288 Does nothing. Context clean up must happen manually
289 since later activities may want to reuse it. Servants using
290 a server connection should cleanup the instance *after* Resources
291 is cleaned up
292 """
293 pass
294
295 -class Server(Ice.Application):
296 """
297 Basic server implementation which can be used for
298 implementing a standalone python server which can
299 be started from icegridnode.
300
301 The servant implementation MUST have a constructor
302 which takes a single ServerContext argument AND
303 have a cleanup() method
304
305 Logging is configured relative to the current directory
306 to be in var/log by default.
307
308 Usage::
309
310 if __name__ == "__main__":
311 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services",""))
312 sys.exit(app.main(sys.argv))
313
314 app.impl now points to an instance of ServicesI
315
316 """
317
318 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
319
320 self.impl_class = impl_class
321 self.adapter_name = adapter_name
322 self.identity = identity
323 self.logdir = logdir
324 self.stop_event = omero.util.concurrency.get_event()
325
326 - def run(self,args):
327
328 from omero.rtypes import ObjectFactories as rFactories
329 from omero.columns import ObjectFactories as cFactories
330
331 props = self.communicator().getProperties()
332 configure_server_logging(props)
333
334 self.logger = logging.getLogger("omero.util.Server")
335 self.logger.info("*"*80)
336 self.logger.info("Starting")
337
338 self.shutdownOnInterrupt()
339
340 try:
341
342 self.objectfactory = omero.clients.ObjectFactory()
343 self.objectfactory.registerObjectFactory(self.communicator())
344 for of in rFactories.values() + cFactories.values():
345 of.register(self.communicator())
346
347 try:
348 serverid = self.communicator().getProperties().getProperty("Ice.ServerId")
349 ctx = ServerContext(serverid, self.communicator(), self.stop_event)
350 self.impl = self.impl_class(ctx)
351 getattr(self.impl, "cleanup")
352 except:
353 self.logger.error("Failed initialization", exc_info=1)
354 sys.exit(100)
355
356 try:
357 self.adapter = self.communicator().createObjectAdapter(self.adapter_name)
358 prx = self.adapter.add(self.impl, self.identity)
359 self.adapter.activate()
360 add_grid_object(self.communicator(), prx)
361 except:
362 self.logger.error("Failed activation", exc_info=1)
363 sys.exit(200)
364
365 self.logger.info("Entering main loop")
366 self.communicator().waitForShutdown()
367 finally:
368 self.stop_event.set()
369 self.logger.info("Cleanup")
370 self.cleanup()
371 self.logger.info("Stopped")
372 self.logger.info("*"*80)
373
375 """
376 Cleans up all resources that were created by this server.
377 Primarily the one servant instance.
378 """
379 if hasattr(self,"impl"):
380 try:
381 self.impl.cleanup()
382 finally:
383 del self.impl
384
387 """
388 Base servant initialization. Doesn't create or try to cleanup
389 a top-level Resources thread. This is useful for large numbers
390 of servants. For servers and other singleton-like servants,
391 see "Servant"
392 """
394 self._lock = threading.RLock()
395 self.ctx = ctx
396 self.stop_event = ctx.stop_event
397 self.communicator = ctx.communicator
398 self.logger = logging.getLogger(make_logname(self))
399 self.logger.debug("Created")
400
402 """
403 Abstract servant which can be used along with a slice2py
404 generated dispatch class as the base type of high-level servants.
405 These provide resource cleanup as per the omero.util.Server
406 class.
407
408 By passing "needs_session = True" to this constructor, an internal
409 session will be created and stored in ServerContext as well as
410 registered with self.resources
411 """
412
413 - def __init__(self, ctx, needs_session = False):
419
421 """
422 Cleanups all resoures created by this servant. Calling
423 cleanup multiple times should be safe.
424 """
425 resources = self.resources
426 self.resources = None
427 if resources != None:
428 self.logger.info("Cleaning up")
429 resources.cleanup()
430 self.logger.info("Done")
431 if self.ctx.hasSession():
432 try:
433 sf = self.ctx.getSession(recreate=False)
434 self.logger.debug("Destroying %s" % sf)
435 sf.destroy()
436 except:
437 pass
438
441
444 """
445 Container class for storing resources which should be
446 cleaned up on close and periodically checked. Use
447 stop_event.set() to stop the internal thread.
448 """
449
450 - def __init__(self, sleeptime = 60, stop_event = None):
451 """
452 Add resources via add(object). They should have a no-arg cleanup()
453 and a check() method.
454
455 The check method will be called periodically (default: 60 seconds)
456 on each resource. The cleanup method will be called on
457 Resources.cleanup()
458 """
459
460 self._lock = threading.RLock()
461 self.logger = logging.getLogger("omero.util.Resources")
462 self.stop_event = stop_event
463 if not self.stop_event:
464 self.stop_event = omero.util.concurrency.get_event()
465
466 if sleeptime < 5:
467 raise exceptions.Exception("Sleep time should be greater than 5: " % sleeptime)
468
469 self.sleeptime = sleeptime
470 self.stuff = []
471
472 class Task(threading.Thread):
473 """
474 Internal thread used for checking "stuff"
475 """
476 def run(self):
477 ctx = self.ctx
478 ctx.logger.info("Starting")
479 while not ctx.stop_event.isSet():
480 try:
481 ctx.logger.debug("Executing")
482 copy = ctx.copyStuff()
483 remove = ctx.checkAll(copy)
484 ctx.removeAll(remove)
485 except:
486 ctx.logger.error("Exception during execution", exc_info = True)
487
488 ctx.logger.debug("Sleeping %s" % ctx.sleeptime)
489
490 try:
491 ctx.stop_event.wait(ctx.sleeptime)
492 except ValueError:
493 pass
494
495 ctx.logger.info("Halted")
496
497 self.thread = Task()
498 self.thread.ctx = self
499 self.thread.start()
500
501 @locked
503 """
504 Within a lock, copy the "stuff" list and reverse it.
505 The list is reversed so that entries added
506 later, which may depend on earlier added entries
507 get a chance to be cleaned up first.
508 """
509 copy = list(self.stuff)
510 copy.reverse()
511 return copy
512
513
515 """
516 While stop_event is unset, go through the copy
517 of stuff and call the check method on each
518 entry. Any that throws an exception or returns
519 a False value will be returned in the remove list.
520 """
521 remove = []
522 for m in copy:
523 if self.stop_event.isSet():
524 return
525 self.logger.debug("Checking %s" % m[0])
526 method = getattr(m[0],m[2])
527 rv = None
528 try:
529 rv = method()
530 except:
531 self.logger.warn("Error from %s" % method, exc_info = True)
532 if not rv:
533 remove.append(m)
534 return remove
535
536 @locked
538 """
539 Finally, within another lock, call the "cleanup"
540 method on all the entries in remove, and remove
541 them from the official stuff list. (If stop_event
542 is set during execution, we return with the assumption
543 that Resources.cleanup() will take care of them)
544 """
545 for r in remove:
546 if self.stop_event.isSet():
547 return
548 self.logger.debug("Removing %s" % r[0])
549 self.safeClean(r)
550 self.stuff.remove(r)
551
552 @locked
553 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
554 entry = (object,cleanupMethod,checkMethod)
555 self.logger.info("Adding object %s" % object)
556 self.stuff.append(entry)
557
558 @locked
560 self.stop_event.set()
561 for m in self.stuff:
562 self.safeClean(m)
563 self.stuff = None
564 self.logger.info("Cleanup done")
565
567 try:
568 self.logger.debug("Cleaning %s" % m[0])
569 method = getattr(m[0],m[1])
570 method()
571 except:
572 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
573
576
578 """
579 Simple class for creating an executable environment
580 """
581
583 """
584 Takes an number of environment variable names which
585 should be copied to the target environment if present
586 in the current execution environment.
587 """
588 if sys.platform == "win32":
589
590 self.env = os.environ.copy()
591 else:
592 self.env = {}
593 for arg in args:
594 if os.environ.has_key(arg):
595 self.env[arg] = os.environ[arg]
597 """
598 Returns the environment map when called.
599 """
600 return self.env
601
602 - def set(self, key, value):
603 """
604 Manually sets a value in the target environment.
605 """
606 self.env[key] = value
607
608 - def append(self, key, addition):
609 """
610 Manually adds a value to the environment string
611 """
612 if self.env.has_key(key):
613 self.env[key] = os.pathsep.join([self.env[key], addition])
614 else:
615 self.set(key, addition)
616