1
2
3
4
5
6
7
8
9 import os
10 import sys
11 import Ice
12 import path
13 import time
14 import uuid
15 import omero
16 import IcePy
17 import IceGrid
18 import logging
19 import platform
20 import Glacier2
21 import threading
22 import exceptions
23 import logging.handlers
24 import omero.util.concurrency
25
26 from omero.util.decorators import locked
27
28 LOGDIR = os.path.join("var","log")
29 LOGFORMAT = """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s"""
30 LOGLEVEL = logging.INFO
31 LOGSIZE = 500000000
32 LOGNUM = 9
33 LOGMODE = "a"
34
35 orig_stdout = sys.stdout
36 orig_stderr = sys.stderr
39 """
40 Generates a logname from the given instance using the module and name from its class
41 """
42 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
43 return log_name
44
68
85
87 """
88 Since all server components should exclusively using the logging module
89 any output to stdout or stderr is caught and logged at "WARN". This is
90 useful, especially in the case of Windows, where stdout/stderr is eaten.
91 """
92
94 self.logger = logger
95 self.internal = logging.getLogger("StreamRedirect")
96 self.softspace = False
97
100
102 msg = msg.strip()
103 if msg:
104 self.logger.warn(msg)
105
107 self.internal.warn("No attribute: %s" % name)
108
109 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
110 """
111 Try to return a ServiceFactory from the grid.
112
113 Try a number of times then give up and raise the
114 last exception returned. This method will only
115 work internally to the grid, i.e. behind the Glacier2
116 firewall. It is intended for internal servers to
117 be able to create sessions for accessing the database. ::
118 communicator := Ice.Communicator used to find the registry
119 user := Username which should have a session created
120 group := Group into which the session should be logged
121 retries := Number of session creation retries before throwing
122 interval := Seconds between retries
123 client_uuid := Uuid of the client which should be used
124 """
125 log = logging.getLogger("omero.utils")
126 if stop_event == None:
127 stop_event = omero.util.concurrency.get_event()
128
129 tryCount = 0
130 excpt = None
131 query = communicator.stringToProxy("IceGrid/Query")
132 query = IceGrid.QueryPrx.checkedCast(query)
133
134 import omero_Constants_ice
135 implicit_ctx = communicator.getImplicitContext()
136 if client_uuid is not None:
137 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
138 else:
139 if not implicit_ctx.containsKey(omero.constants.CLIENTUUID):
140 client_uuid = str(uuid.uuid4())
141 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
142
143 while tryCount < retries:
144 if stop_event.isSet():
145 return None
146 try:
147 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0]
148 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz)
149 sf = blitz.create(user, None)
150
151 return omero.api.ServiceFactoryPrx.checkedCast(sf)
152 except Exception, e:
153 tryCount += 1
154 log.info("Failed to get session on attempt %s", str(tryCount))
155 excpt = e
156 stop_event.wait(interval)
157
158 log.warn("Reason: %s", str(excpt))
159 raise excpt
160
162 """
163 """
164 reg = communicator.stringToProxy("IceGrid/Registry")
165 reg = IceGrid.RegistryPrx.checkedCast(reg)
166 adm = reg.createAdminSession('null', '')
167 return adm
168
170 """
171 """
172 sid = communicator.identityToString(obj.ice_getIdentity())
173 adm = create_admin_session(communicator)
174 prx = adm.getAdmin()
175 try:
176 try:
177 prx.addObject(obj)
178 except IceGrid.ObjectExistsException:
179 prx.updateObject(obj)
180 finally:
181 adm.destroy()
182
184 """
185 Converts a long to a path such that for all directiories only
186 a 1000 files and a 1000 subdirectories will be returned.
187
188 This method duplicates the logic in
189 ome.io.nio.AbstractFileSystemService.java:getPath()
190 """
191 suffix = ""
192 remaining = id
193 dirno = 0
194
195 if id is None or id == "":
196 raise exceptions.Exception("Expecting a not-null id.")
197
198 id = long(id)
199
200 if id < 0:
201 raise exceptions.Exception("Expecting a non-negative id.")
202
203 while (remaining > 999):
204 remaining /= 1000
205
206 if remaining > 0:
207 dirno = remaining % 1000
208 suffix = os.path.join("Dir-%03d" % dirno, suffix)
209
210 return os.path.join(root, "%s%s" %(suffix,id))
211
212 -class ServerContext(object):
213 """
214 Context passed to all servants.
215
216 server_id, communicator, and stop_event will be
217 constructed by the top-level Server instance.
218
219 A context instance may also be configured to hold
220 on to an internal session (ServiceFactoryPrx) and
221 keep it alive.
222
223 This instance obeys the Resources API and calls
224 sf.keepAlive(None) on every check call, but does
225 nothing on cleanup. The sf instance must be manually
226 cleaned as the final operation of a servant.
227
228 (Note: cleanup of the server context indicates
229 server shutdown, so should be infrequent)
230 """
231
232 - def __init__(self, server_id, communicator, stop_event, on_newsession = None):
233 self._lock = threading.RLock()
234 self.logger = logging.getLogger("omero.util.ServerContext")
235 self.server_id = server_id
236 self.communicator = communicator
237 self.stop_event = stop_event
238 self.servant_map = dict()
239 self.on_newsession = None
240
241 @locked
242 - def add_servant(self, adapter_or_current, servant, ice_identity = None):
243 oa = adapter_or_current
244 if isinstance(adapter_or_current, (Ice.Current, IcePy.Current)):
245 oa = oa.adapter
246 if ice_identity is None:
247 prx = oa.addWithUUID(servant)
248 else:
249 prx = oa.add(servant, ice_identity)
250
251 servant.setProxy(prx)
252 self.servant_map[prx] = servant
253 return prx
254
255 - def newSession(self):
256 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event)
257 if callable(self.on_newsession):
258 self.on_newsession(self.session)
259
260 - def hasSession(self):
261 return hasattr(self, "session")
262
263 @locked
264 - def getSession(self, recreate = True):
265 """
266 Returns the ServiceFactoryPrx configured for the context if
267 available. If the context was not configured for sessions,
268 an ApiUsageException will be thrown: servants should know
269 whether or not they were configured for sessions.
270 See Servant(..., needs_session = True)
271
272 Otherwise, if there is no ServiceFactoryPrx, an attempt will
273 be made to create one if recreate == True. If the value is None
274 or non can be recreated, an InternalException will be thrown.
275
276 TODO : currently no arguments are provided for re-creating these,
277 but also not in Servant.__init__
278 """
279 if not self.hasSession():
280 raise omero.ApiUsageException("Not configured for server connection")
281
282 if self.session:
283 try:
284 self.session.keepAlive(None)
285 except Ice.CommunicatorDestroyedException:
286 self.session = None
287 except exceptions.Exception, e:
288 self.logger.warn("Connection failure: %s" % e)
289 self.session = None
290
291 if self.session is None and recreate:
292 try:
293 self.newSession()
294 self.logger.info("Established connection: %s" % self.session)
295 except exceptions.Exception, e:
296 self.logger.warn("Failed to establish connection: %s" % e)
297
298 if self.session is None:
299 raise omero.InternalException("No connection to server")
300
301 return self.session
302
304 """
305 Calls getSession() but always returns True. This keeps the context
306 available in the resources for later uses, and tries to re-establish
307 a connection in case Blitz goes down.
308 """
309 try:
310 self.getSession()
311 except:
312 pass
313 return True
314
316 """
317 Does nothing. Context clean up must happen manually
318 since later activities may want to reuse it. Servants using
319 a server connection should cleanup the instance *after* Resources
320 is cleaned up
321 """
322 pass
323
324 -class Server(Ice.Application):
325 """
326 Basic server implementation which can be used for
327 implementing a standalone python server which can
328 be started from icegridnode.
329
330 The servant implementation MUST have a constructor
331 which takes a single ServerContext argument AND
332 have a cleanup() method
333
334 Logging is configured relative to the current directory
335 to be in var/log by default.
336
337 Usage::
338
339 if __name__ == "__main__":
340 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services",""))
341 sys.exit(app.main(sys.argv))
342
343 app.impl now points to an instance of ServicesI
344
345 """
346
347 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
348
349 self.impl_class = impl_class
350 self.adapter_name = adapter_name
351 self.identity = identity
352 self.logdir = logdir
353 self.stop_event = omero.util.concurrency.get_event()
354
355 - def run(self,args):
402
404 """
405 Cleans up all resources that were created by this server.
406 Primarily the one servant instance.
407 """
408 if hasattr(self,"impl"):
409 try:
410 self.impl.cleanup()
411 finally:
412 del self.impl
413
416 """
417 Base servant initialization. Doesn't create or try to cleanup
418 a top-level Resources thread. This is useful for large numbers
419 of servants. For servers and other singleton-like servants,
420 see "Servant"
421 """
423 self._lock = threading.RLock()
424 self.prx = None
425 self.ctx = ctx
426 self.stop_event = ctx.stop_event
427 self.communicator = ctx.communicator
428 self.logger = logging.getLogger(make_logname(self))
429 self.logger.debug("Created")
430
432 """
433 Should be overwritten for post-initialization activities.
434 The reason this method exists is that the implementation
435 must be complete before registering it with the adapter.
436 """
437 self.prx = prx
438
440 """
441 Abstract servant which can be used along with a slice2py
442 generated dispatch class as the base type of high-level servants.
443 These provide resource cleanup as per the omero.util.Server
444 class.
445
446 By passing "needs_session = True" to this constructor, an internal
447 session will be created and stored in ServerContext as well as
448 registered with self.resources
449 """
450
451 - def __init__(self, ctx, needs_session = False):
457
459 """
460 Cleanups all resoures created by this servant. Calling
461 cleanup multiple times should be safe.
462 """
463 resources = self.resources
464 self.resources = None
465 if resources != None:
466 self.logger.info("Cleaning up")
467 resources.cleanup()
468 self.logger.info("Done")
469 if self.ctx.hasSession():
470 try:
471 sf = self.ctx.getSession(recreate=False)
472 self.logger.debug("Destroying %s" % sf)
473 sf.destroy()
474 except:
475 pass
476
479
482 """
483 Container class for storing resources which should be
484 cleaned up on close and periodically checked. Use
485 stop_event.set() to stop the internal thread.
486 """
487
488 - def __init__(self, sleeptime = 60, stop_event = None):
489 """
490 Add resources via add(object). They should have a no-arg cleanup()
491 and a check() method.
492
493 The check method will be called periodically (default: 60 seconds)
494 on each resource. The cleanup method will be called on
495 Resources.cleanup()
496 """
497
498 self._lock = threading.RLock()
499 self.logger = logging.getLogger("omero.util.Resources")
500 self.stop_event = stop_event
501 if not self.stop_event:
502 self.stop_event = omero.util.concurrency.get_event()
503
504 if sleeptime < 5:
505 raise exceptions.Exception("Sleep time should be greater than 5: " % sleeptime)
506
507 self.sleeptime = sleeptime
508 self.stuff = []
509
510 class Task(threading.Thread):
511 """
512 Internal thread used for checking "stuff"
513 """
514 def run(self):
515 ctx = self.ctx
516 ctx.logger.info("Starting")
517 while not ctx.stop_event.isSet():
518 try:
519 ctx.logger.debug("Executing")
520 copy = ctx.copyStuff()
521 remove = ctx.checkAll(copy)
522 ctx.removeAll(remove)
523 except:
524 ctx.logger.error("Exception during execution", exc_info = True)
525
526 ctx.logger.debug("Sleeping %s" % ctx.sleeptime)
527
528 try:
529 ctx.stop_event.wait(ctx.sleeptime)
530 except ValueError:
531 pass
532
533 ctx.logger.info("Halted")
534
535 self.thread = Task()
536 self.thread.ctx = self
537 self.thread.start()
538
539 @locked
541 """
542 Within a lock, copy the "stuff" list and reverse it.
543 The list is reversed so that entries added
544 later, which may depend on earlier added entries
545 get a chance to be cleaned up first.
546 """
547 copy = list(self.stuff)
548 copy.reverse()
549 return copy
550
551
553 """
554 While stop_event is unset, go through the copy
555 of stuff and call the check method on each
556 entry. Any that throws an exception or returns
557 a False value will be returned in the remove list.
558 """
559 remove = []
560 for m in copy:
561 if self.stop_event.isSet():
562 return
563 self.logger.debug("Checking %s" % m[0])
564 method = getattr(m[0],m[2])
565 rv = None
566 try:
567 rv = method()
568 except:
569 self.logger.warn("Error from %s" % method, exc_info = True)
570 if not rv:
571 remove.append(m)
572 return remove
573
574 @locked
576 """
577 Finally, within another lock, call the "cleanup"
578 method on all the entries in remove, and remove
579 them from the official stuff list. (If stop_event
580 is set during execution, we return with the assumption
581 that Resources.cleanup() will take care of them)
582 """
583 for r in remove:
584 if self.stop_event.isSet():
585 return
586 self.logger.debug("Removing %s" % r[0])
587 self.safeClean(r)
588 self.stuff.remove(r)
589
590 @locked
591 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
592 entry = (object,cleanupMethod,checkMethod)
593 self.logger.debug("Adding object %s" % object)
594 self.stuff.append(entry)
595
596 @locked
598 self.stop_event.set()
599 for m in self.stuff:
600 self.safeClean(m)
601 self.stuff = None
602 self.logger.debug("Cleanup done")
603
605 try:
606 self.logger.debug("Cleaning %s" % m[0])
607 method = getattr(m[0],m[1])
608 method()
609 except:
610 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
611
614
616 """
617 Simple class for creating an executable environment
618 """
619
621 """
622 Takes an number of environment variable names which
623 should be copied to the target environment if present
624 in the current execution environment.
625 """
626 if sys.platform == "win32":
627
628 self.env = os.environ.copy()
629 else:
630 self.env = {}
631 for arg in args:
632 if os.environ.has_key(arg):
633 self.env[arg] = os.environ[arg]
635 """
636 Returns the environment map when called.
637 """
638 return self.env
639
640 - def set(self, key, value):
641 """
642 Manually sets a value in the target environment.
643 """
644 self.env[key] = value
645
646 - def append(self, key, addition):
647 """
648 Manually adds a value to the environment string
649 """
650 if self.env.has_key(key):
651 self.env[key] = os.pathsep.join([self.env[key], addition])
652 else:
653 self.set(key, addition)
654
660 try:
661 from win32com.shell import shellcon, shell
662 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
663 except ImportError:
664 homeprop = os.path.expanduser("~")
665 return homeprop
666
668 f = path.path(path_or_obj)
669 editor = os.getenv("VISUAL") or os.getenv("EDITOR")
670 if not editor:
671 if platform.system() == "Windows":
672 editor = "Notepad.exe"
673 else:
674 editor = "vi"
675 f.write_text(start_text)
676 from which import which
677 editor_path = which(editor)
678 pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f)
679 if pid:
680 re = RuntimeError("Couldn't spawn editor: %s" % editor)
681 re.pid = pid
682 raise re
683
684
685 -def tail_lines(filename,linesback=10,returnlist=0):
686 """Does what "tail -10 filename" would have done
687 Parameters::
688 filename file to read
689 linesback Number of lines to read from end of file
690 returnlist Return a list containing the lines instead of a string
691
692 """
693 avgcharsperline=75
694
695 file = open(filename,'r')
696 while 1:
697 try: file.seek(-1 * avgcharsperline * linesback,2)
698 except IOError: file.seek(0)
699 if file.tell() == 0: atstart=1
700 else: atstart=0
701
702 lines=file.read().split("\n")
703 if (len(lines) > (linesback+1)) or atstart: break
704
705 avgcharsperline=avgcharsperline * 1.3
706 file.close()
707
708 if len(lines) > linesback: start=len(lines)-linesback -1
709 else: start=0
710 if returnlist: return lines[start:len(lines)-1]
711
712 out=""
713 for l in lines[start:len(lines)-1]: out=out + l + "\n"
714 return out
715