1
2
3
4
5
6
7
8
9 import os
10 import sys
11 import Ice
12 import path
13 import time
14 import uuid
15 import omero
16 import IcePy
17 import IceGrid
18 import logging
19 import platform
20 import Glacier2
21 import threading
22 import exceptions
23 import logging.handlers
24 import omero.util.concurrency
25
26 from omero.util.decorators import locked
27
28 LOGDIR = os.path.join("var","log")
29 LOGFORMAT = """%(asctime)s %(levelname)-5.5s [%(name)40s] (%(threadName)-10s) %(message)s"""
30 LOGLEVEL = logging.INFO
31 LOGSIZE = 500000000
32 LOGNUM = 9
33 LOGMODE = "a"
34
35 orig_stdout = sys.stdout
36 orig_stderr = sys.stderr
39 """
40 Generates a logname from the given instance using the module and name from its class
41 """
42 log_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
43 return log_name
44
68
85
87 """
88 Since all server components should exclusively using the logging module
89 any output to stdout or stderr is caught and logged at "WARN". This is
90 useful, especially in the case of Windows, where stdout/stderr is eaten.
91 """
92
94 self.logger = logger
95 self.internal = logging.getLogger("StreamRedirect")
96 self.softspace = False
97
100
102 msg = msg.strip()
103 if msg:
104 self.logger.warn(msg)
105
108
109 -def internal_service_factory(communicator, user="root", group=None, retries=6, interval=10, client_uuid=None, stop_event = None):
110 """
111 Try to return a ServiceFactory from the grid.
112
113 Try a number of times then give up and raise the
114 last exception returned. This method will only
115 work internally to the grid, i.e. behind the Glacier2
116 firewall. It is intended for internal servers to
117 be able to create sessions for accessing the database. ::
118 communicator := Ice.Communicator used to find the registry
119 user := Username which should have a session created
120 group := Group into which the session should be logged
121 retries := Number of session creation retries before throwing
122 interval := Seconds between retries
123 client_uuid := Uuid of the client which should be used
124 """
125 log = logging.getLogger("omero.utils")
126 if stop_event == None:
127 stop_event = omero.util.concurrency.get_event(name="internal_service_factory")
128
129 tryCount = 0
130 excpt = None
131 query = communicator.stringToProxy("IceGrid/Query")
132 query = IceGrid.QueryPrx.checkedCast(query)
133
134 import omero_Constants_ice
135 implicit_ctx = communicator.getImplicitContext()
136 if client_uuid is not None:
137 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
138 else:
139 if not implicit_ctx.containsKey(omero.constants.CLIENTUUID):
140 client_uuid = str(uuid.uuid4())
141 implicit_ctx.put(omero.constants.CLIENTUUID, client_uuid)
142
143 while tryCount < retries:
144 if stop_event.isSet():
145 return None
146 try:
147 blitz = query.findAllObjectsByType("::Glacier2::SessionManager")[0]
148 blitz = Glacier2.SessionManagerPrx.checkedCast(blitz)
149 sf = blitz.create(user, None)
150
151 return omero.api.ServiceFactoryPrx.checkedCast(sf)
152 except Exception, e:
153 tryCount += 1
154 log.info("Failed to get session on attempt %s", str(tryCount))
155 excpt = e
156 stop_event.wait(interval)
157
158 log.warn("Reason: %s", str(excpt))
159 raise excpt
160
162 """
163 """
164 reg = communicator.stringToProxy("IceGrid/Registry")
165 reg = IceGrid.RegistryPrx.checkedCast(reg)
166 adm = reg.createAdminSession('null', '')
167 return adm
168
170 """
171 """
172 sid = communicator.identityToString(obj.ice_getIdentity())
173 adm = create_admin_session(communicator)
174 prx = adm.getAdmin()
175 try:
176 try:
177 prx.addObject(obj)
178 except IceGrid.ObjectExistsException:
179 prx.updateObject(obj)
180 finally:
181 adm.destroy()
182
184 """
185 Converts a long to a path such that for all directiories only
186 a 1000 files and a 1000 subdirectories will be returned.
187
188 This method duplicates the logic in
189 ome.io.nio.AbstractFileSystemService.java:getPath()
190 """
191 suffix = ""
192 remaining = id
193 dirno = 0
194
195 if id is None or id == "":
196 raise exceptions.Exception("Expecting a not-null id.")
197
198 id = long(id)
199
200 if id < 0:
201 raise exceptions.Exception("Expecting a non-negative id.")
202
203 while (remaining > 999):
204 remaining /= 1000
205
206 if remaining > 0:
207 dirno = remaining % 1000
208 suffix = os.path.join("Dir-%03d" % dirno, suffix)
209
210 return os.path.join(root, "%s%s" %(suffix,id))
211
212 -class ServerContext(object):
213 """
214 Context passed to all servants.
215
216 server_id, communicator, and stop_event will be
217 constructed by the top-level Server instance.
218
219 A context instance may also be configured to hold
220 on to an internal session (ServiceFactoryPrx) and
221 keep it alive.
222
223 This instance obeys the Resources API and calls
224 sf.keepAlive(None) on every check call, but does
225 nothing on cleanup. The sf instance must be manually
226 cleaned as the final operation of a servant.
227
228 (Note: cleanup of the server context indicates
229 server shutdown, so should be infrequent)
230 """
231
232 - def __init__(self, server_id, communicator, stop_event, on_newsession = None):
233 self._lock = threading.RLock()
234 self.logger = logging.getLogger("omero.util.ServerContext")
235 self.server_id = server_id
236 self.communicator = communicator
237 self.stop_event = stop_event
238 self.servant_map = dict()
239 self.on_newsession = None
240
241 @locked
242 - def add_servant(self, adapter_or_current, servant, ice_identity = None):
243 oa = adapter_or_current
244 if isinstance(adapter_or_current, (Ice.Current, IcePy.Current)):
245 oa = oa.adapter
246 if ice_identity is None:
247 prx = oa.addWithUUID(servant)
248 else:
249 prx = oa.add(servant, ice_identity)
250
251 servant.setProxy(prx)
252 self.servant_map[prx] = servant
253 return prx
254
255 - def newSession(self):
256 self.session = internal_service_factory(self.communicator, stop_event = self.stop_event)
257 if callable(self.on_newsession):
258 self.on_newsession(self.session)
259
260 - def hasSession(self):
261 return hasattr(self, "session")
262
263 @locked
264 - def getSession(self, recreate = True):
265 """
266 Returns the ServiceFactoryPrx configured for the context if
267 available. If the context was not configured for sessions,
268 an ApiUsageException will be thrown: servants should know
269 whether or not they were configured for sessions.
270 See Servant(..., needs_session = True)
271
272 Otherwise, if there is no ServiceFactoryPrx, an attempt will
273 be made to create one if recreate == True. If the value is None
274 or non can be recreated, an InternalException will be thrown.
275
276 TODO : currently no arguments are provided for re-creating these,
277 but also not in Servant.__init__
278 """
279 if not self.hasSession():
280 raise omero.ApiUsageException("Not configured for server connection")
281
282 if self.session:
283 try:
284 self.session.keepAlive(None)
285 except Ice.CommunicatorDestroyedException:
286 self.session = None
287 except exceptions.Exception, e:
288 self.logger.warn("Connection failure: %s" % e)
289 self.session = None
290
291 if self.session is None and recreate:
292 try:
293 self.newSession()
294 self.logger.info("Established connection: %s" % self.session)
295 except exceptions.Exception, e:
296 self.logger.warn("Failed to establish connection: %s" % e)
297
298 if self.session is None:
299 raise omero.InternalException("No connection to server")
300
301 return self.session
302
304 """
305 Calls getSession() but always returns True. This keeps the context
306 available in the resources for later uses, and tries to re-establish
307 a connection in case Blitz goes down.
308 """
309 try:
310 self.getSession()
311 except:
312 pass
313 return True
314
316 """
317 Does nothing. Context clean up must happen manually
318 since later activities may want to reuse it. Servants using
319 a server connection should cleanup the instance *after* Resources
320 is cleaned up
321 """
322 pass
323
324 -class Server(Ice.Application):
325 """
326 Basic server implementation which can be used for
327 implementing a standalone python server which can
328 be started from icegridnode.
329
330 The servant implementation MUST have a constructor
331 which takes a single ServerContext argument AND
332 have a cleanup() method
333
334 Logging is configured relative to the current directory
335 to be in var/log by default.
336
337 Usage::
338
339 if __name__ == "__main__":
340 app=Server(ServicesI, "ServicesAdapter", Ice.Identity("Services",""))
341 sys.exit(app.main(sys.argv))
342
343 app.impl now points to an instance of ServicesI
344
345 """
346
347 - def __init__(self, impl_class, adapter_name, identity, logdir = LOGDIR):
348
349 self.impl_class = impl_class
350 self.adapter_name = adapter_name
351 self.identity = identity
352 self.logdir = logdir
353 self.stop_event = omero.util.concurrency.get_event(name="Server")
354
355 - def run(self,args):
402
404 """
405 Cleans up all resources that were created by this server.
406 Primarily the one servant instance.
407 """
408 if hasattr(self,"impl"):
409 try:
410 self.impl.cleanup()
411 finally:
412 del self.impl
413
416 """
417 Base servant initialization. Doesn't create or try to cleanup
418 a top-level Resources thread. This is useful for large numbers
419 of servants. For servers and other singleton-like servants,
420 see "Servant"
421 """
423 self._lock = threading.RLock()
424 self.prx = None
425 self.ctx = ctx
426 self.stop_event = ctx.stop_event
427 self.communicator = ctx.communicator
428 self.logger = logging.getLogger(make_logname(self))
429 self.logger.debug("Created")
430
432 """
433 Should be overwritten for post-initialization activities.
434 The reason this method exists is that the implementation
435 must be complete before registering it with the adapter.
436 """
437 self.prx = prx
438
440 """
441 Abstract servant which can be used along with a slice2py
442 generated dispatch class as the base type of high-level servants.
443 These provide resource cleanup as per the omero.util.Server
444 class.
445
446 By passing "needs_session = True" to this constructor, an internal
447 session will be created and stored in ServerContext as well as
448 registered with self.resources
449 """
450
451 - def __init__(self, ctx, needs_session = False):
457
459 """
460 Cleanups all resoures created by this servant. Calling
461 cleanup multiple times should be safe.
462 """
463 resources = self.resources
464 self.resources = None
465 if resources != None:
466 self.logger.info("Cleaning up")
467 resources.cleanup()
468 self.logger.info("Done")
469 if self.ctx.hasSession():
470 try:
471 sf = self.ctx.getSession(recreate=False)
472 self.logger.debug("Destroying %s" % sf)
473 sf.destroy()
474 except:
475 pass
476
479
482 """
483 Container class for storing resources which should be
484 cleaned up on close and periodically checked. Use
485 stop_event.set() to stop the internal thread.
486 """
487
488 - def __init__(self, sleeptime = 60, stop_event = None):
489 """
490 Add resources via add(object). They should have a no-arg cleanup()
491 and a check() method.
492
493 The check method will be called periodically (default: 60 seconds)
494 on each resource. The cleanup method will be called on
495 Resources.cleanup()
496 """
497
498 self.stuff = []
499 self._lock = threading.RLock()
500 self.logger = logging.getLogger("omero.util.Resources")
501 self.stop_event = stop_event
502 if not self.stop_event:
503 self.stop_event = omero.util.concurrency.get_event(name="Resources")
504
505 if sleeptime < 5:
506 raise exceptions.Exception("Sleep time should be greater than 5: %s" % sleeptime)
507
508 self.sleeptime = sleeptime
509
510 class Task(threading.Thread):
511 """
512 Internal thread used for checking "stuff"
513 """
514 def run(self):
515 ctx = self.ctx
516 ctx.logger.info("Starting")
517 while not ctx.stop_event.isSet():
518 try:
519 ctx.logger.debug("Executing")
520 copy = ctx.copyStuff()
521 remove = ctx.checkAll(copy)
522 ctx.removeAll(remove)
523 except:
524 ctx.logger.error("Exception during execution", exc_info = True)
525
526 ctx.logger.debug("Sleeping %s" % ctx.sleeptime)
527
528 try:
529 ctx.stop_event.wait(ctx.sleeptime)
530 except ValueError:
531 pass
532
533 if isinstance(ctx.stop_event, omero.util.concurrency.AtExitEvent):
534 if ctx.stop_event.atexit:
535 return
536
537 ctx.logger.info("Halted")
538
539 self.thread = Task()
540 self.thread.ctx = self
541 self.thread.start()
542
543 @locked
545 """
546 Within a lock, copy the "stuff" list and reverse it.
547 The list is reversed so that entries added
548 later, which may depend on earlier added entries
549 get a chance to be cleaned up first.
550 """
551 copy = list(self.stuff)
552 copy.reverse()
553 return copy
554
555
557 """
558 While stop_event is unset, go through the copy
559 of stuff and call the check method on each
560 entry. Any that throws an exception or returns
561 a False value will be returned in the remove list.
562 """
563 remove = []
564 for m in copy:
565 if self.stop_event.isSet():
566 return
567 self.logger.debug("Checking %s" % m[0])
568 method = getattr(m[0],m[2])
569 rv = None
570 try:
571 rv = method()
572 except:
573 self.logger.warn("Error from %s" % method, exc_info = True)
574 if not rv:
575 remove.append(m)
576 return remove
577
578 @locked
580 """
581 Finally, within another lock, call the "cleanup"
582 method on all the entries in remove, and remove
583 them from the official stuff list. (If stop_event
584 is set during execution, we return with the assumption
585 that Resources.cleanup() will take care of them)
586 """
587 for r in remove:
588 if self.stop_event.isSet():
589 return
590 self.logger.debug("Removing %s" % r[0])
591 self.safeClean(r)
592 self.stuff.remove(r)
593
594 @locked
595 - def add(self, object, cleanupMethod = "cleanup", checkMethod = "check"):
596 entry = (object,cleanupMethod,checkMethod)
597 self.logger.debug("Adding object %s" % object)
598 self.stuff.append(entry)
599
600 @locked
602 self.stop_event.set()
603 for m in self.stuff:
604 self.safeClean(m)
605 self.stuff = None
606 self.logger.debug("Cleanup done")
607
609 try:
610 self.logger.debug("Cleaning %s" % m[0])
611 method = getattr(m[0],m[1])
612 method()
613 except:
614 self.logger.error("Error cleaning resource: %s" % m[0], exc_info=1)
615
618
620 """
621 Simple class for creating an executable environment
622 """
623
625 """
626 Takes an number of environment variable names which
627 should be copied to the target environment if present
628 in the current execution environment.
629 """
630 if sys.platform == "win32":
631
632 self.env = os.environ.copy()
633 else:
634 self.env = {}
635 for arg in args:
636 if os.environ.has_key(arg):
637 self.env[arg] = os.environ[arg]
639 """
640 Returns the environment map when called.
641 """
642 return self.env
643
644 - def set(self, key, value):
645 """
646 Manually sets a value in the target environment.
647 """
648 self.env[key] = value
649
650 - def append(self, key, addition):
651 """
652 Manually adds a value to the environment string
653 """
654 if self.env.has_key(key):
655 self.env[key] = os.pathsep.join([self.env[key], addition])
656 else:
657 self.set(key, addition)
658
664 exceptions_to_handle = (ImportError)
665 try:
666 from pywintypes import com_error
667 from win32com.shell import shellcon, shell
668 exceptions_to_handle = (ImportError, com_error)
669 homeprop = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
670 except exceptions_to_handle:
671 homeprop = os.path.expanduser("~")
672 return homeprop
673
675 f = path.path(path_or_obj)
676 editor = os.getenv("VISUAL") or os.getenv("EDITOR")
677 if not editor:
678 if platform.system() == "Windows":
679 editor = "Notepad.exe"
680 else:
681 editor = "vi"
682 f.write_text(start_text)
683 from which import which
684 editor_path = which(editor)
685 pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f)
686 if pid:
687 re = RuntimeError("Couldn't spawn editor: %s" % editor)
688 re.pid = pid
689 raise re
690
691
692 -def tail_lines(filename,linesback=10,returnlist=0):
693 """Does what "tail -10 filename" would have done
694 Parameters::
695 filename file to read
696 linesback Number of lines to read from end of file
697 returnlist Return a list containing the lines instead of a string
698
699 """
700 avgcharsperline=75
701
702 file = open(filename,'r')
703 while 1:
704 try: file.seek(-1 * avgcharsperline * linesback,2)
705 except IOError: file.seek(0)
706 if file.tell() == 0: atstart=1
707 else: atstart=0
708
709 lines=file.read().split("\n")
710 if (len(lines) > (linesback+1)) or atstart: break
711
712 avgcharsperline=avgcharsperline * 1.3
713 file.close()
714
715 if len(lines) > linesback: start=len(lines)-linesback -1
716 else: start=0
717 if returnlist: return lines[start:len(lines)-1]
718
719 out=""
720 for l in lines[start:len(lines)-1]: out=out + l + "\n"
721 return out
722