1
2
3
4
5
6
7
8
9 import os
10 import sys
11 import Ice
12 import path
13 import time
14 import omero
15 import IcePy
16 import IceGrid
17 import logging
18 import platform
19 import Glacier2
20 import threading
21 import exceptions
22 import logging.handlers
23 import omero.util.concurrency
24 import omero_ext.uuid as uuid
25
26 from omero.util.decorators import locked
27
28 LOGDIR = os.path.join("var","log")
29 LOGFORMAT = """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s"""
30 LOGLEVEL = logging.INFO
31 LOGSIZE = 500000000
32 LOGNUM = 9
33 LOGMODE = "a"
34
35 orig_stdout = sys.stdout
36 orig_stderr = sys.stderr
39 """
40 Generates a logname from the given instance using the module and name from its class
41 """
42 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
43 return log_name
44
68
85
87 """
88 Since all server components should exclusively using the logging module
89 any output to stdout or stderr is caught and logged at "WARN". This is
90 useful, especially in the case of Windows, where stdout/stderr is eaten.
91 """
92
94 self.logger = logger
95 self.internal = logging.getLogger("StreamRedirect")
96 self.softspace = False
97
100
102 msg = msg.strip()
103 if msg:
104 self.logger.warn(msg)
105
108
110 """
111 Centralized logic for declaring and logging a service
112 dependency on a non-shipped library. This is called
113 lazily from the run method of the application to give
114 logging time to be initialized.
115
116 See #4566
117 """
118
121
123 """
124 Get version method which returns a string
125 representing. Should be overwritten by
126 subclasses for packages/modules with no
127 __version__ field.
128 """
129 return target.__version__
130
131 - def check(self, logger):
132 try:
133 target = __import__(self.key)
134 version = self.get_version(target)
135 logger.info("Loaded dependency %s (%s)" % (self.key, version))
136 return True
137 except ImportError:
138 logger.error("Failed to load: '%s'" % self.key)
139 return False
140
141 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
142 """
143 Try to return a ServiceFactory from the grid.
144
145 Try a number of times then give up and raise the
146 last exception returned. This method will only
147 work internally to the grid, i.e. behind the Glacier2
148 firewall. It is intended for internal servers to
149 be able to create sessions for accessing the database. ::
150 communicator := Ice.Communicator used to find the registry
151 user := Username which should have a session created
152 group := Group into which the session should be logged
153 retries := Number of session creation retries before throwing
154 interval := Seconds between retries
155 client_uuid := Uuid of the client which should be used
156 """
157 log = logging.getLogger("omero.utils")
158 if stop_event == None:
159 stop_event = omero.util.concurrency.get_event(name="internal_service_factory")
160
161 tryCount = 0
162 excpt = None
163 query = communicator.stringToProxy("IceGrid/Query")
164 query = IceGrid.QueryPrx.checkedCast(query)
165
166 import omero_Constants_ice
167 implicit_ctx = communicator.getImplicitContext()
168 implicit_ctx.put(omero.constants.AGENT, "Python service")
169 if client_uuid is not None:
170 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
171 else:
172 if not implicit_ctx.containsKey(omero.constants.CLIENTUUID):
173 client_uuid = str(uuid.uuid4())
174 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
175
176 while tryCount < retries:
177 if stop_event.isSet():
178 return None
179 try:
180 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0]
181 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz)
182 sf = blitz.create(user, None)
183
184 return omero.api.ServiceFactoryPrx.checkedCast(sf)
185 except Exception, e:
186 tryCount += 1
187 log.info("Failed to get session on attempt %s", str(tryCount))
188 excpt = e
189 stop_event.wait(interval)
190
191 log.warn("Reason: %s", str(excpt))
192 if excpt:
193 raise excpt
194
196 """
197 """
198 reg = communicator.stringToProxy("IceGrid/Registry")
199 reg = IceGrid.RegistryPrx.checkedCast(reg)
200 adm = reg.createAdminSession('null', '')
201 return adm
202
204 """
205 """
206 sid = communicator.identityToString(obj.ice_getIdentity())
207 adm = create_admin_session(communicator)
208 prx = adm.getAdmin()
209 try:
210 try:
211 prx.addObject(obj)
212 except IceGrid.ObjectExistsException:
213 prx.updateObject(obj)
214 finally:
215 adm.destroy()
216
218 """
219 Converts a long to a path such that for all directiories only
220 a 1000 files and a 1000 subdirectories will be returned.
221
222 This method duplicates the logic in
223 ome.io.nio.AbstractFileSystemService.java:getPath()
224 """
225 suffix = ""
226 remaining = id
227 dirno = 0
228
229 if id is None or id == "":
230 raise exceptions.Exception("Expecting a not-null id.")
231
232 id = long(id)
233
234 if id < 0:
235 raise exceptions.Exception("Expecting a non-negative id.")
236
237 while (remaining > 999):
238 remaining /= 1000
239
240 if remaining > 0:
241 dirno = remaining % 1000
242 suffix = os.path.join("Dir-%03d" % dirno, suffix)
243
244 return os.path.join(root, "%s%s" %(suffix,id))
245
246 -class ServerContext(object):
247 """
248 Context passed to all servants.
249
250 server_id, communicator, and stop_event will be
251 constructed by the top-level Server instance.
252
253 A context instance may also be configured to hold
254 on to an internal session (ServiceFactoryPrx) and
255 keep it alive.
256
257 This instance obeys the Resources API and calls
258 sf.keepAlive(None) on every check call, but does
259 nothing on cleanup. The sf instance must be manually
260 cleaned as the final operation of a servant.
261
262 (Note: cleanup of the server context indicates
263 server shutdown, so should be infrequent)
264 """
265
266 - def __init__(self, server_id, communicator, stop_event, on_newsession = None):
267 self._lock = threading.RLock()
268 self.logger = logging.getLogger("omero.util.ServerContext")
269 self.server_id = server_id
270 self.communicator = communicator
271 self.stop_event = stop_event
272 self.servant_map = dict()
273 self.on_newsession = None
274
275 @locked
276 - def add_servant(self, adapter_or_current, servant, ice_identity = None):
277 oa = adapter_or_current
278 if isinstance(adapter_or_current, (Ice.Current, IcePy.Current)):
279 oa = oa.adapter
280 if ice_identity is None:
281 prx = oa.addWithUUID(servant)
282 else:
283 prx = oa.add(servant, ice_identity)
284
285 servant.setProxy(prx)
286 self.servant_map[prx] = servant
287 return prx
288
289 - def newSession(self):
290 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event)
291 if callable(self.on_newsession):
292 self.on_newsession(self.session)
293
294 - def hasSession(self):
295 return hasattr(self, "session")
296
297 @locked
298 - def getSession(self, recreate = True):
299 """
300 Returns the ServiceFactoryPrx configured for the context if
301 available. If the context was not configured for sessions,
302 an ApiUsageException will be thrown: servants should know
303 whether or not they were configured for sessions.
304 See Servant(..., needs_session = True)
305
306 Otherwise, if there is no ServiceFactoryPrx, an attempt will
307 be made to create one if recreate == True. If the value is None
308 or non can be recreated, an InternalException will be thrown.
309
310 TODO : currently no arguments are provided for re-creating these,
311 but also not in Servant.__init__
312 """
313 if not self.hasSession():
314 raise omero.ApiUsageException("Not configured for server connection")
315
316 if self.session:
317 try:
318 self.session.keepAlive(None)
319 except Ice.CommunicatorDestroyedException:
320 self.session = None
321 except exceptions.Exception, e:
322 self.logger.warn("Connection failure: %s" % e)
323 self.session = None
324
325 if self.session is None and recreate:
326 try:
327 self.newSession()
328 self.logger.info("Established connection: %s" % self.session)
329 except exceptions.Exception, e:
330 self.logger.warn("Failed to establish connection: %s" % e)
331
332 if self.session is None:
333 raise omero.InternalException("No connection to server")
334
335 return self.session
336
338 """
339 Calls getSession() but always returns True. This keeps the context
340 available in the resources for later uses, and tries to re-establish
341 a connection in case Blitz goes down.
342 """
343 try:
344 self.getSession()
345 except:
346 pass
347 return True
348
350 """
351 Does nothing. Context clean up must happen manually
352 since later activities may want to reuse it. Servants using
353 a server connection should cleanup the instance *after* Resources
354 is cleaned up
355 """
356 pass
357
358 -class Server(Ice.Application):
359 """
360 Basic server implementation which can be used for
361 implementing a standalone python server which can
362 be started from icegridnode.
363
364 The servant implementation MUST have a constructor
365 which takes a single ServerContext argument AND
366 have a cleanup() method
367
368 Logging is configured relative to the current directory
369 to be in var/log by default.
370
371 Usage::
372
373 if __name__ == "__main__":
374 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services",""))
375 sys.exit(app.main(sys.argv))
376
377 app.impl now points to an instance of ServicesI
378
379 """
380
381 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR, dependencies = ()):
389
390 - def run(self,args):
446
448 """
449 Cleans up all resources that were created by this server.
450 Primarily the one servant instance.
451 """
452 if hasattr(self,"impl"):
453 try:
454 self.impl.cleanup()
455 finally:
456 del self.impl
457
460 """
461 Base servant initialization. Doesn't create or try to cleanup
462 a top-level Resources thread. This is useful for large numbers
463 of servants. For servers and other singleton-like servants,
464 see "Servant"
465 """
467 self._lock = threading.RLock()
468 self.prx = None
469 self.ctx = ctx
470 self.stop_event = ctx.stop_event
471 self.communicator = ctx.communicator
472 self.logger = logging.getLogger(make_logname(self))
473 self.logger.debug("Created")
474
476 """
477 Should be overwritten for post-initialization activities.
478 The reason this method exists is that the implementation
479 must be complete before registering it with the adapter.
480 """
481 self.prx = prx
482
484 """
485 Abstract servant which can be used along with a slice2py
486 generated dispatch class as the base type of high-level servants.
487 These provide resource cleanup as per the omero.util.Server
488 class.
489
490 By passing "needs_session = True" to this constructor, an internal
491 session will be created and stored in ServerContext as well as
492 registered with self.resources
493 """
494
495 - def __init__(self, ctx, needs_session = False):
501
503 """
504 Cleanups all resoures created by this servant. Calling
505 cleanup multiple times should be safe.
506 """
507 resources = self.resources
508 self.resources = None
509 if resources != None:
510 self.logger.info("Cleaning up")
511 resources.cleanup()
512 self.logger.info("Done")
513 if self.ctx.hasSession():
514 try:
515 sf = self.ctx.getSession(recreate=False)
516 self.logger.debug("Destroying %s" % sf)
517 sf.destroy()
518 except:
519 pass
520
523
526 """
527 Container class for storing resources which should be
528 cleaned up on close and periodically checked. Use
529 stop_event.set() to stop the internal thread.
530 """
531
532 - def __init__(self, sleeptime = 60, stop_event = None):
533 """
534 Add resources via add(object). They should have a no-arg cleanup()
535 and a check() method.
536
537 The check method will be called periodically (default: 60 seconds)
538 on each resource. The cleanup method will be called on
539 Resources.cleanup()
540 """
541
542 self.stuff = []
543 self._lock = threading.RLock()
544 self.logger = logging.getLogger("omero.util.Resources")
545 self.stop_event = stop_event
546 if not self.stop_event:
547 self.stop_event = omero.util.concurrency.get_event(name="Resources")
548
549 if sleeptime < 5:
550 raise exceptions.Exception("Sleep time should be greater than 5: %s" % sleeptime)
551
552 self.sleeptime = sleeptime
553
554 class Task(threading.Thread):
555 """
556 Internal thread used for checking "stuff"
557 """
558 def run(self):
559 ctx = self.ctx
560 ctx.logger.info("Starting")
561 while not ctx.stop_event.isSet():
562 try:
563 ctx.logger.debug("Executing")
564 copy = ctx.copyStuff()
565 remove = ctx.checkAll(copy)
566 ctx.removeAll(remove)
567 except:
568 ctx.logger.error("Exception during execution", exc_info = True)
569
570 ctx.logger.debug("Sleeping %s" % ctx.sleeptime)
571
572 try:
573 ctx.stop_event.wait(ctx.sleeptime)
574 except ValueError:
575 pass
576
577 if isinstance(ctx.stop_event, omero.util.concurrency.AtExitEvent):
578 if ctx.stop_event.atexit:
579 return
580
581 ctx.logger.info("Halted")
582
583 self.thread = Task()
584 self.thread.ctx = self
585 self.thread.start()
586
587 @locked
589 """
590 Within a lock, copy the "stuff" list and reverse it.
591 The list is reversed so that entries added
592 later, which may depend on earlier added entries
593 get a chance to be cleaned up first.
594 """
595 copy = list(self.stuff)
596 copy.reverse()
597 return copy
598
599
601 """
602 While stop_event is unset, go through the copy
603 of stuff and call the check method on each
604 entry. Any that throws an exception or returns
605 a False value will be returned in the remove list.
606 """
607 remove = []
608 for m in copy:
609 if self.stop_event.isSet():
610 return
611 self.logger.debug("Checking %s" % m[0])
612 method = getattr(m[0],m[2])
613 rv = None
614 try:
615 rv = method()
616 except:
617 self.logger.warn("Error from %s" % method, exc_info = True)
618 if not rv:
619 remove.append(m)
620 return remove
621
622 @locked
624 """
625 Finally, within another lock, call the "cleanup"
626 method on all the entries in remove, and remove
627 them from the official stuff list. (If stop_event
628 is set during execution, we return with the assumption
629 that Resources.cleanup() will take care of them)
630 """
631 for r in remove:
632 if self.stop_event.isSet():
633 return
634 self.logger.debug("Removing %s" % r[0])
635 self.safeClean(r)
636 self.stuff.remove(r)
637
638 @locked
639 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
643
644 @locked
646 self.stop_event.set()
647 for m in self.stuff:
648 self.safeClean(m)
649 self.stuff = None
650 self.logger.debug("Cleanup done")
651
653 try:
654 self.logger.debug("Cleaning %s" % m[0])
655 method = getattr(m[0],m[1])
656 method()
657 except:
658 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
659
662
664 """
665 Simple class for creating an executable environment
666 """
667
669 """
670 Takes an number of environment variable names which
671 should be copied to the target environment if present
672 in the current execution environment.
673 """
674 if sys.platform == "win32":
675
676 self.env = os.environ.copy()
677 else:
678 self.env = {}
679 for arg in args:
680 if os.environ.has_key(arg):
681 self.env[arg] = os.environ[arg]
683 """
684 Returns the environment map when called.
685 """
686 return self.env
687
688 - def set(self, key, value):
689 """
690 Manually sets a value in the target environment.
691 """
692 self.env[key] = value
693
694 - def append(self, key, addition):
695 """
696 Manually adds a value to the environment string
697 """
698 if self.env.has_key(key):
699 self.env[key] = os.pathsep.join([self.env[key], addition])
700 else:
701 self.set(key, addition)
702
703
704
705
706
707 -def get_user(default = None):
708 """
709 Returns the username. For most purposes, this value
710 will be the same as getpass.getuser on *nix and
711 win32api.GetUserName on Windows, but in some situations
712 (when running without a terminal, etc) getuser may throw
713 a KeyError. In which case, or if the username resolves to
714 False, the default value be returned.
715
716 Any unexpected exceptions will be thrown.
717
718 See ticket:6307
719 """
720 rv = None
721 try:
722 import getpass
723 rv = getpass.getuser()
724 except KeyError:
725 pass
726 except ImportError:
727 import win32api
728 rv = win32api.GetUserName()
729
730 if not rv:
731 return default
732 else:
733 return rv
734
737 exceptions_to_handle = (ImportError)
738 try:
739 from pywintypes import com_error
740 from win32com.shell import shellcon, shell
741 exceptions_to_handle = (ImportError, com_error)
742 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
743 except exceptions_to_handle:
744 homeprop = os.path.expanduser("~")
745
746 if "~" == homeprop:
747 raise exceptions.Exception("Unexpanded '~' from expanduser: see ticket:5583")
748
749 return homeprop
750
752 f = path.path(path_or_obj)
753 editor = os.getenv("VISUAL") or os.getenv("EDITOR")
754 if not editor:
755 if platform.system() == "Windows":
756 editor = "Notepad.exe"
757 else:
758 editor = "vi"
759 f.write_text(start_text)
760
761
762
763
764 editor_obj = path.path(editor)
765 if editor_obj.isabs():
766 editor_path = editor
767 else:
768 from omero_ext.which import which
769 editor_path = which(editor)
770
771 pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f)
772 if pid:
773 re = RuntimeError("Couldn't spawn editor: %s" % editor)
774 re.pid = pid
775 raise re
776
777
778 -def tail_lines(filename,linesback=10,returnlist=0):
779 """Does what "tail -10 filename" would have done
780 Parameters::
781 filename file to read
782 linesback Number of lines to read from end of file
783 returnlist Return a list containing the lines instead of a string
784
785 """
786 avgcharsperline=75
787
788 file = open(filename,'r')
789 while 1:
790 try: file.seek(-1 * avgcharsperline * linesback,2)
791 except IOError: file.seek(0)
792 if file.tell() == 0: atstart=1
793 else: atstart=0
794
795 lines=file.read().split("\n")
796 if (len(lines) > (linesback+1)) or atstart: break
797
798 avgcharsperline=avgcharsperline * 1.3
799 file.close()
800
801 if len(lines) > linesback: start=len(lines)-linesback -1
802 else: start=0
803 if returnlist: return lines[start:len(lines)-1]
804
805 out=""
806 for l in lines[start:len(lines)-1]: out=out + l + "\n"
807 return out
808