Package omero :: Module scripts
[hide private]
[frames] | no frames]

Source Code for Module omero.scripts

  1  #!/usr/bin/env python 
  2  """ 
  3     Scripting types 
  4         - Classes: 
  5             - Type        --  Top of parameter type hierarchy 
  6             - Long        -- 
  7             - String      -- 
  8             - Bool        -- 
  9             - List  
 10             - Map 
 11             - Set  
 12         - Functions: 
 13             - client      -- Produces an omero.client object with given input/output constraints. 
 14   
 15     Copyright 2008 Glencoe Software, Inc. All rights reserved. 
 16     Use is subject to license terms supplied in LICENSE.txt 
 17   
 18  """ 
 19   
 20  import os 
 21  import Ice 
 22  import logging 
 23  import exceptions 
 24   
 25  import omero 
 26  import omero_Scripts_ice 
 27  import omero.callbacks 
 28  import omero.util.concurrency 
 29  import omero.util.temp_files 
 30   
 31  from omero.rtypes import * 
 32   
 33   
 34  TYPE_LOG = logging.getLogger("omero.scripts.Type") 
 35  PROC_LOG = logging.getLogger("omero.scripts.ProcessCallback") 
 36   
 37   
38 -class Type(omero.grid.Param):
39 """ 40 omero.grid.Param subclass which provides convenience methods for input/output specification. 41 Further subclasses are responsible for creating proper prototypes. 42 43 kwargs 44 """ 45 PROTOTYPE_FUNCTION = None 46 PROTOTYPE_DEFAULT = None 47 PROTOTYPE_MIN = None 48 PROTOTYPE_MAX = None 49 PROTOTYPE_VALUES = None 50
51 - def __init__(self, name, optional = True, out = False, description = None, default = None, **kwargs):
52 53 omero.grid.Param.__init__(self) 54 55 # Non-Param attributes 56 self._name = name 57 self._in = True 58 self._out = out 59 60 # Other values will be filled in by the kwargs 61 # Mostly leaving these for backwards compatibility 62 self.description = description 63 self.optional = optional 64 65 # Assign all the kwargs 66 for k, v in kwargs.items(): 67 if not hasattr(self, k): 68 TYPE_LOG.warn("Unknown property: %s", k) 69 setattr(self, k, v) 70 71 _DEF = self.__get(self.PROTOTYPE_DEFAULT, False) 72 _FUN = self.__get(self.PROTOTYPE_FUNCTION) 73 _MAX = self.__get(self.PROTOTYPE_MAX) 74 _MIN = self.__get(self.PROTOTYPE_MIN) 75 _VAL = self.__get(self.PROTOTYPE_VALUES) 76 77 # Someone specifically set the prototype, then 78 # we assume that useDefault should be True 79 if default is not None: # For whatever reason, inheritance isn't working. 80 newfunc = _FUN 81 newdefault = default 82 if isinstance(self, List): 83 if isinstance(default, (list, tuple)): 84 newdefault = wrap(default).val 85 elif isinstance(default, omero.RCollection): 86 newfunc = lambda x: x 87 elif isinstance(default, omero.RType): 88 default = [default] 89 else: 90 newfunc = lambda x: x 91 newdefault = rlist([rtype(default)]) 92 self.useDefault = True 93 self.prototype = newfunc(newdefault) 94 else: 95 if not callable(_FUN): 96 raise ValueError("Bad prototype function: %s" % _FUN) 97 98 # To prevent weirdness, if the class default is 99 # callable, we'll assume its a constructor and 100 # create a new one to prevent modification. 101 try: 102 _def = _DEF() 103 except TypeError: 104 _def = _DEF 105 self.prototype = _FUN(_def) 106 107 # The following use wrap to guarantee that an rtype is present 108 if self.min is not None: 109 if _MIN is None: 110 self.min = _FUN(self.min) 111 else: 112 self.min = _MIN(self.min) 113 114 if self.max is not None: 115 if _MAX is None: 116 self.max = _FUN(self.max) 117 else: 118 self.max = _MAX(self.max) 119 120 if self.values is not None: 121 if _VAL is None: 122 self.values = wrap(self.values) 123 else: 124 self.values = _VAL(self.values) 125 126 # Now if useDefault has been set, either manually, or 127 # via setting default="..." we check that the default 128 # value matches a value if present 129 if self.values is not None and self.values and self.useDefault: 130 if isinstance(self.prototype, omero.RCollection): 131 test = unwrap(self.prototype.val[0]) 132 else: 133 test = unwrap(self.prototype) 134 values = unwrap(self.values) 135 if test not in values: 136 raise ValueError("%s is not in %s" % (test, values))
137 138
139 - def out(self):
140 self._in = False 141 self._out = True 142 return self
143
144 - def inout(self):
145 self._in = True 146 self._out = True 147 return self
148
149 - def type(self, *arg):
150 self.prototype = wrap(arg) 151 return self
152
153 - def __get(self, val, func = True):
154 if val is not None: 155 if func: 156 return val.im_func 157 else: 158 return val
159 160
161 -class Long(Type):
162 """ 163 Wraps an rlong 164 """ 165 PROTOTYPE_FUNCTION = rlong 166 PROTOTYPE_DEFAULT = 0
167 168
169 -class Int(Type):
170 """ 171 Wraps an rint 172 """ 173 PROTOTYPE_FUNCTION = rint 174 PROTOTYPE_DEFAULT = 0
175 176
177 -class Double(Type):
178 """ 179 Wraps an rdouble 180 """ 181 PROTOTYPE_FUNCTION = rdouble 182 PROTOTYPE_DEFAULT = 0.0
183 184
185 -class Float(Type):
186 """ 187 Wraps an rfloat 188 """ 189 PROTOTYPE_FUNCTION = rfloat 190 PROTOTYPE_DEFAULT = 0.0
191 192
193 -class String(Type):
194 """ 195 Wraps an rstring 196 """ 197 PROTOTYPE_FUNCTION = rstring 198 PROTOTYPE_DEFAULT = ""
199 200
201 -class Bool(Type):
202 """ 203 Wraps an rbool 204 """ 205 PROTOTYPE_FUNCTION = rbool 206 PROTOTYPE_DEFAULT = False
207 208
209 -class Color(Type):
210 """ 211 Wraps an rinternal(Color) 212 """ 213 PROTOTYPE_FUNCTION = rinternal 214 PROTOTYPE_DEFAULT = omero.Color
215 216
217 -class Point(Type):
218 """ 219 Wraps an rinternal(Point) 220 """ 221 PROTOTYPE_FUNCTION = rinternal 222 PROTOTYPE_FUNCTION = omero.Point
223 224
225 -class Plane(Type):
226 """ 227 Wraps an rinternal(Plane) 228 """ 229 PROTOTYPE_FUNCTION = rinternal 230 PROTOTYPE_DEFAULT = omero.Plane
231 232
233 -class __Coll(Type):
234 """ 235 Base type providing the append and extend functionality. 236 Not for user use. 237 """ 238 PROTOTYPE_DEFAULT = list 239
240 - def append(self, *arg):
241 self.prototype.val.append(*arg) 242 return self
243
244 - def extend(self, *arg):
245 self.prototype.val.extend(*arg) 246 return self
247
248 - def ofType(self, obj):
249 if callable(obj): 250 obj = obj() # Ctors, etc. 251 252 # If someone used default=, then "ofType()" is not necessary 253 # and so we check for their correspondence. 254 if self.useDefault and self.prototype.val: 255 if not isinstance(obj, self.prototype.val[0].__class__): 256 raise ValueError("ofType values doesn't match default value: %s <> %s" % (unwrap(obj), unwrap(self.prototype.val[0]))) 257 else: 258 self.prototype.val.append(wrap(obj)) 259 260 return self
261
262 -class Set(__Coll):
263 """ 264 Wraps an rset. To add values to the contents of the set, 265 use "append" or "extend" since set.val is of type list. 266 """ 267 PROTOTYPE_FUNCTION = rset
268 269
270 -class List(__Coll):
271 """ 272 Wraps an rlist. To add values to the contents of the list, 273 use "append" or "extend" since set.val is of type list. 274 """ 275 PROTOTYPE_FUNCTION = rlist
276 277
278 -class Map(Type):
279 """ 280 Wraps an rmap. To add values to the contents of the map, 281 use "update" since map.val is of type dict. 282 """ 283 PROTOTYPE_FUNCTION = rmap 284 PROTOTYPE_DEFAULT = dict 285
286 - def update(self, *args, **kwargs):
287 self.prototype.val.update(*args, **kwargs) 288 return self
289 290
291 -class ParseExit(exceptions.Exception):
292 """ 293 Raised when this script should just parse parameters and return. 294 """ 295
296 - def __init__(self, params):
297 exceptions.Exception.__init__(self) 298 self.params = params
299 300
301 -def client(*args, **kwargs):
302 """ 303 Entry point for all script engine scripts. 304 305 Typical usage consists of:: 306 307 client = omero.scripts.client("name","description", \ 308 omero.scripts.Long("name"),...) 309 310 where the returned client is created via the empty constructor to omero.client 311 using only --Ice.Config or ICE_CONFIG, and the function arguments are taken 312 as metdata about the current script. With this information, all script 313 consumers should be able to determine the required types for execution. 314 315 Possible types are all subclasses of omero.scripts.Type 316 317 To change the omero.model.Format of the stdout and stderr produced by 318 this script, use the constructor arguments:: 319 320 client = omero.scripts.client(..., \ 321 stdoutFormat = "text/plain", 322 stderrFormat = "text/plain") 323 324 If you would like to prevent stdout and/or stderr from being 325 uploaded, set the corresponding value to None. If you would like 326 to overwrite the value with another file, use 327 client.setOutput(). Though it is possible to attach any RType to 328 "stdout" or "stderr", there is an assumption that the value will 329 be an robject(OriginalFileI()) 330 331 Providing your own client is possible via the kwarg "client = ...", 332 but be careful since this may break usage with the rest of the 333 scripting framework. The client should not have a session, and 334 must be configured for the argumentless version of createSession() 335 """ 336 337 args = list(args) 338 if len(args) >= 1: 339 if isinstance(args[0], str): 340 kwargs["name"] = args.pop(0) 341 if len(args) >= 1: 342 if isinstance(args[0], str): 343 kwargs["description"] = args.pop(0) 344 345 if not kwargs.has_key("client"): 346 kwargs["client"] = omero.client() 347 c = kwargs["client"] 348 c.setAgent("OMERO.scripts") 349 350 if args and isinstance(args[0], omero.grid.JobParams): 351 c.params = args.pop(0) 352 else: 353 c.params = omero.grid.JobParams() 354 c.params.inputs = {} 355 c.params.outputs = {} 356 357 for k, v in kwargs.items(): 358 if hasattr(c.params, k): 359 setattr(c.params, k, v) 360 361 if not c.params.stdoutFormat: 362 c.params.stdoutFormat = "text/plain" 363 364 if not c.params.stderrFormat: 365 c.params.stderrFormat = "text/plain" 366 367 for p in args: 368 if isinstance(p, Type): 369 if p._in: 370 c.params.inputs[p._name] = p 371 if p._out: 372 c.params.outputs[p._name] = p 373 else: 374 raise ValueError("Not Type: %s" % type(p)) 375 376 handleParse(c) # May throw 377 378 c.createSession().detachOnDestroy() 379 return c
380
381 -def handleParse(c):
382 """ 383 Raises ParseExit if the client has the configuration property 384 "omero.scripts.parse". If the value is anything other than "only", 385 then the parameters will also be sent to the server. 386 """ 387 parse = c.getProperty("omero.scripts.parse") 388 if len(parse) > 0: # TODO Add to omero/Constants.ice 389 if parse != "only": 390 c.createSession().detachOnDestroy() 391 c.setOutput("omero.scripts.parse", rinternal(c.params)) 392 raise ParseExit(c.params)
393
394 -def parse_text(scriptText):
395 """ 396 Parses the given script text with "omero.scripts.parse" set 397 and catches the exception. The parameters are returned. 398 399 WARNING: This method calls "exec" on the given text. 400 Do NOT use this on data you don't trust. 401 """ 402 try: 403 cfg = omero.util.temp_files.create_path() 404 cfg.write_lines(["omero.scripts.parse=only", "omero.host=localhost"]) 405 old = os.environ.get("ICE_CONFIG") 406 try: 407 os.environ["ICE_CONFIG"] = cfg.abspath() 408 exec(scriptText, {"__name__":"__main__"}) 409 raise exceptions.Exception("Did not throw ParseExit: %s" % scriptText) 410 finally: 411 if old: 412 os.environ["ICE_CONFIG"] = old 413 except ParseExit, exit: 414 return exit.params
415
416 -def parse_file(filename):
417 """ 418 Parses the given script file with "omero.scripts.parse" set 419 and catches the exception. The parameters are returned. 420 421 WARNING: This method calls "exec" on the given file's contents. 422 Do NOT use this on data you don't trust. 423 """ 424 from path import path 425 scriptText = path(filename).text() 426 return parse_text(scriptText)
427 428
429 -class MissingInputs(exceptions.Exception):
430 - def __init__(self):
431 exceptions.Exception.__init__(self) 432 self.keys = []
433 434
435 -def parse_inputs(inputs_strings, params):
436 """ 437 Parses a command-line like string representation of input parameters 438 into an RDict (a map from string to RType). The input should be an 439 iterable of arguments of the form "key=value". 440 441 For example, "a=1" gets mapped to {"a":1} 442 """ 443 inputs = {} 444 for input_string in inputs_strings: 445 rv = parse_input(input_string, params) 446 inputs.update(rv) 447 448 missing = MissingInputs() 449 for key, param in params.inputs.items(): 450 a = inputs.get(key, None) 451 if not a: 452 if param.useDefault: 453 inputs[key] = param.prototype 454 elif not param.optional: 455 missing.keys.append(key) 456 457 if missing.keys: 458 missing.inputs = inputs 459 raise missing 460 461 return inputs
462 463
464 -def parse_input(input_string, params):
465 """ 466 Parse a single input_string. The params 467 """ 468 parts = input_string.split("=") 469 if len(parts) == 1: 470 parts.append("") 471 key = parts[0] 472 val = parts[1] 473 474 param = params.inputs.get(key) 475 if param is None: 476 return {} 477 elif isinstance(param.prototype,\ 478 (omero.RLong, omero.RString, omero.RInt, omero.RBool,\ 479 omero.RTime, omero.RDouble, omero.RFloat)): 480 val = param.prototype.__class__(val) 481 elif isinstance(param.prototype, omero.RList): 482 items = val.split(",") 483 if len(param.prototype.val) == 0: 484 # Don't know what needs to be added here, so calling wrap 485 # which will produce an rlist of rstrings. 486 val = omero.rtypes.wrap(items) 487 else: 488 p = param.prototype.val[0] 489 val = omero.rtypes.rlist([p.__class__(x) for x in items]) 490 else: 491 raise ValueError("No converter for: %s (type=%s)" % (key, param.prototype.__class__)) 492 493 return {key:val}
494 495
496 -def group_params(params):
497 """ 498 Walks through the inputs of the given JobParams 499 and returns a map-of-maps with Param names as 500 the leaf nodes. 501 502 For example, for the following: 503 504 Params("1", grouping = "A") # "A." is equivalent 505 Params("2", grouping = "A.B") 506 Params("3", grouping = "A.C") 507 508 this function returns: 509 510 {"A" {"": "1" : "B" : "2", "C" : "3"} } 511 512 while: 513 514 Params("1", grouping = "A") 515 516 returns: 517 518 {"A" : "1"} 519 520 """ 521 groupings = dict() 522 for k, v in params.inputs.items(): 523 524 val = v.grouping 525 if not val.endswith("."): 526 val = val + "." 527 528 parts = val.split(".") 529 530 g = groupings 531 while parts: 532 p = parts.pop(0) 533 try: 534 g = g[p] 535 except KeyError: 536 if parts: 537 g[p] = dict() 538 g = g[p] 539 else: 540 g[p] = k 541 542 # Now find all subtrees of the form {"": "key"} and 543 # replace them by themselves 544 tuples = [(groupings, k, v) for k, v in groupings.items()] 545 while tuples: 546 new_tuples = [] 547 for g, k, v in tuples: 548 if isinstance(v, dict): 549 if len(v) == 1 and "" in v: 550 g[k] = v[""] 551 else: 552 new_tuples.extend([(v, k2, v2) for k2, v2 in v.items()]) 553 tuples = new_tuples 554 555 return groupings
556
557 -def error_msg(category, key, format_string, *args):
558 c = "%s" % (category.upper()) 559 s = """%s for "%s": %s\n""" % (c, key, format_string) 560 return s % args
561
562 -def compare_proto(key, proto, input, cache=None):
563 564 if cache is None: 565 cache = {} 566 567 if id(proto) in cache and id(input) in cache: 568 return "" # Prevent StackOverflow 569 else: 570 cache[id(proto)] = True 571 cache[id(input)] = True 572 573 itype = input is None and None or input.__class__ 574 ptype = proto is None and None or proto.__class__ 575 576 if not isinstance(input, ptype): 577 return error_msg("Wrong type", key, "%s != %s", itype, ptype) 578 579 # Now recurse if a collection type 580 errors = "" 581 if isinstance(proto, omero.RMap) and len(proto.val) > 0: 582 for x in input.val.values(): 583 errors += compare_proto(key, proto.val.values()[0], x, cache) 584 elif isinstance(proto, omero.RCollection) and len(proto.val) > 0: 585 for x in input.val: 586 errors += compare_proto(key, proto.val[0], x, cache) 587 return errors
588
589 -def expand(input):
590 if input is None: 591 items = [] 592 elif isinstance(input, (list, tuple)): 593 items = list(input) 594 elif isinstance(input, dict): 595 items = input.values() 596 else: 597 items = [input] 598 return items
599
600 -def check_boundaries(key, min, max, input):
601 errors = "" 602 603 # Unwrap 604 min = unwrap(min) 605 max = unwrap(max) 606 input = unwrap(input) 607 items = expand(input) 608 609 # Check 610 for x in items: 611 if min is not None and min > x: 612 errors += error_msg("Out of bounds", key, "%s is below min %s", x, min) 613 if max is not None and max < x: 614 errors += error_msg("Out of bounds", key, "%s is above max %s", x, max) 615 return errors
616
617 -def check_values(key, values, input):
618 errors = "" 619 620 # Unwrap 621 values = unwrap(values) 622 input = unwrap(input) 623 items = expand(input) 624 values = expand(values) 625 626 if not values: 627 return errors 628 629 for x in items: 630 if x not in values: 631 errors += error_msg("Value list", key, "%s not in %s", x, values) 632 633 return errors
634
635 -def validate_inputs(params, inputs, svc = None, session = None):
636 """ 637 Method used by processor.py to check the input values 638 provided by the user launching the script. If a non-empty 639 errors string is returned, then the inputs fail validation. 640 641 A service instance can be provided in order to add default 642 values to the session. If no service instance is provided, 643 values with a default which are missing will be counted as 644 errors. 645 """ 646 errors = "" 647 for key, param in params.inputs.items(): 648 if key not in inputs: 649 if param.optional: 650 if param.useDefault and svc is not None: 651 ignore = set_input(svc, session, key, param.prototype) 652 else: # Not optional 653 if param.useDefault: 654 errors += set_input(svc, session, key, param.prototype) 655 else: 656 errors += error_msg("Missing input", key, "") 657 else: 658 input = inputs[key] 659 errors += compare_proto(key, param.prototype, input) 660 errors += check_boundaries(key, param.min, param.max, input) 661 errors += check_values(key, param.values, input) 662 return errors
663
664 -def set_input(svc, session, key, value):
665 try: 666 svc.setInput(session, key, value) 667 return "" 668 except exceptions.Exception, e: 669 return error_msg("Failed to set intput", key, "%s=%s. Error: %s", key, value, e)
670 671 # 672 # Importing into omero.scripts namespace 673 # 674 ProcessCallbackI = omero.callbacks.ProcessCallbackI 675
676 -def wait(client, process, ms = 500):
677 """ 678 Wrapper around the use of ProcessCallbackI 679 """ 680 cb = ProcessCallbackI(client, process) 681 while cb.block(500) is None: 682 process.poll()
683