Package omero :: Module columns
[hide private]
[frames] | no frames]

Source Code for Module omero.columns

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  """
 
  4  ::
 
  5      /*
 
  6       *   $Id$
 
  7       *
 
  8       *   Copyright 2009 Glencoe Software, Inc. All rights reserved.
 
  9       *   Use is subject to license terms supplied in LICENSE.txt
 
 10       *
 
 11       */
 
 12  """ 
 13  
 
 14  """
 
 15  Concrete implementations of the omero.grid.Column
 
 16  type hierarchy which know how to convert themselves
 
 17  to PyTables types.
 
 18  """ 
 19  
 
 20  import omero, Ice, IceImport 
 21  IceImport.load("omero_Tables_ice") 
 22  
 
 23  try: 
 24      import numpy 
 25      tables = __import__("tables") # Pytables 
 26      has_pytables = True 
 27  except ImportError: 
 28      has_pytables = False 
 29  
 
 30  
 
31 -def columns2definition(cols):
32 """ 33 Takes a list of columns and converts them into a map 34 from names to tables.* column descriptors 35 """ 36 definition = {} 37 for i in range(len(cols)): 38 column = cols[i] 39 instance = column.descriptor(pos=i) 40 if column.name in definition: 41 raise omero.ApiUsageException( 42 None, None, "Duplicate column name: %s" % column.name) 43 definition[column.name] = instance 44 # Descriptions are handled separately 45 return definition
46
47 -class AbstractColumn(object):
48 """ 49 Base logic for all columns 50 """ 51
52 - def __init__(self):
53 # Note: don't rely on any properties such as self.name being set if 54 # this has been called through Ice 55 d = self.descriptor(None) 56 if isinstance(d, tables.IsDescription): 57 cols = d.columns 58 try: 59 del cols["_v_pos"] 60 except KeyError: 61 pass 62 self._types = [None] * len(cols) 63 self._subnames = [None] * len(cols) 64 for k, v in cols.items(): 65 self._types[v._v_pos] = v.recarrtype 66 self._subnames[v._v_pos] = "/" + k 67 68 else: 69 self._types = [d.recarrtype] 70 self._subnames = [""]
71
72 - def settable(self, tbl):
73 """ 74 Called by tables.py when first initializing columns. 75 Can be used to complete further initialization. 76 """ 77 self.__table = tbl
78
79 - def append(self, tbl):
80 """ 81 Called by tables.py to give columns. By default, does nothing. 82 """ 83 pass
84
85 - def readCoordinates(self, tbl, rowNumbers):
86 if rowNumbers is None or len(rowNumbers) == 0: 87 rows = tbl.read() 88 else: 89 rows = tbl.readCoordinates(rowNumbers) 90 self.fromrows(rows)
91
92 - def read(self, tbl, start, stop):
93 rows = tbl.read(start, stop) 94 self.fromrows(rows)
95
96 - def getsize(self):
97 """ 98 Any method which does not use the "values" field 99 will need to override this method. 100 """ 101 if self.values is None: 102 return None 103 else: 104 return len(self.values)
105
106 - def setsize(self, size):
107 """ 108 Any method which does not use the "values" field 109 will need to override this method. 110 """ 111 if size is None: 112 self.values = None 113 else: 114 self.values = [None for x in range(size)]
115
116 - def arrays(self):
117 """ 118 Any method which does not use the "values" field 119 will need to override this method. 120 """ 121 return [self.values]
122
123 - def dtypes(self):
124 """ 125 Override this method if descriptor() doesn't return the correct data 126 type/size at initialisation- this is mostly a problem for array types 127 """ 128 names = [self.name + sn for sn in self._subnames] 129 return zip(names, self._types)
130
131 - def fromrows(self, rows):
132 """ 133 Any method which does not use the "values" field 134 will need to override this method. 135 """ 136 self.values = rows[self.name] 137 # WORKAROUND: http://www.zeroc.com/forums/bug-reports/4165-icepy-can-not-handle-buffers-longs-i64.html#post20468 138 # see ticket:1951 and #2160 139 ## d = self.recarrtypes[0][1] 140 ## Disabled until Ice 3.4 141 ## if isinstance(d, str): 142 ## d = numpy.dtype(d) 143 ## if d.kind == "S" or (d.kind == "i" and d.itemsize == "8"): 144 self.values = self.values.tolist()
145
146 -class FileColumnI(AbstractColumn, omero.grid.FileColumn):
147
148 - def __init__(self, name = "Unknown", *args):
149 omero.grid.FileColumn.__init__(self, name, *args) 150 AbstractColumn.__init__(self)
151
152 - def descriptor(self, pos):
153 return tables.Int64Col(pos=pos)
154
155 -class ImageColumnI(AbstractColumn, omero.grid.ImageColumn):
156
157 - def __init__(self, name = "Unknown", *args):
158 omero.grid.ImageColumn.__init__(self, name, *args) 159 AbstractColumn.__init__(self)
160
161 - def descriptor(self, pos):
162 return tables.Int64Col(pos=pos)
163
164 -class WellColumnI(AbstractColumn, omero.grid.WellColumn):
165
166 - def __init__(self, name = "Unknown", *args):
167 omero.grid.WellColumn.__init__(self, name, *args) 168 AbstractColumn.__init__(self)
169
170 - def descriptor(self, pos):
171 return tables.Int64Col(pos=pos)
172
173 -class PlateColumnI(AbstractColumn, omero.grid.PlateColumn):
174
175 - def __init__(self, name = "Unknown", *args):
176 omero.grid.PlateColumn.__init__(self, name, *args) 177 AbstractColumn.__init__(self)
178
179 - def descriptor(self, pos):
180 return tables.Int64Col(pos=pos)
181
182 -class RoiColumnI(AbstractColumn, omero.grid.RoiColumn):
183
184 - def __init__(self, name = "Unknown", *args):
185 omero.grid.RoiColumn.__init__(self, name, *args) 186 AbstractColumn.__init__(self)
187
188 - def descriptor(self, pos):
189 return tables.Int64Col(pos=pos)
190
191 -class BoolColumnI(AbstractColumn, omero.grid.BoolColumn):
192
193 - def __init__(self, name = "Unknown", *args):
194 omero.grid.BoolColumn.__init__(self, name, *args) 195 AbstractColumn.__init__(self)
196
197 - def descriptor(self, pos):
198 return tables.BoolCol(pos=pos)
199
200 -class DoubleColumnI(AbstractColumn, omero.grid.DoubleColumn):
201
202 - def __init__(self, name = "Unknown", *args):
203 omero.grid.DoubleColumn.__init__(self, name, *args) 204 AbstractColumn.__init__(self)
205
206 - def descriptor(self, pos):
207 return tables.Float64Col(pos=pos)
208
209 -class LongColumnI(AbstractColumn, omero.grid.LongColumn):
210
211 - def __init__(self, name = "Unknown", *args):
212 omero.grid.LongColumn.__init__(self, name, *args) 213 AbstractColumn.__init__(self)
214
215 - def descriptor(self, pos):
216 return tables.Int64Col(pos=pos)
217
218 -class StringColumnI(AbstractColumn, omero.grid.StringColumn):
219
220 - def __init__(self, name = "Unknown", *args):
221 omero.grid.StringColumn.__init__(self, name, *args) 222 AbstractColumn.__init__(self)
223
224 - def settable(self, tbl):
225 AbstractColumn.settable(self, tbl) 226 self.size = getattr(tbl.cols, self.name).dtype.itemsize
227
228 - def arrays(self):
229 """ 230 Check for strings longer than the initialised column width 231 """ 232 for v in self.values: 233 if len(v) > self.size: 234 raise omero.ValidationException( 235 None, None, "Maximum string length in column %s is %d" % 236 (self.name, self.size)) 237 return [self.values]
238
239 - def dtypes(self):
240 """ 241 Overriding to correct for size. 242 (Testing suggests this may not be necessary, the size appears to be 243 correctly set at initialisation) 244 """ 245 return [(self.name, "S", self.size)]
246
247 - def descriptor(self, pos):
248 # During initialization, size might be zero 249 # to prevent exceptions we temporarily assume size 1 250 if pos is None: 251 return tables.StringCol(pos=pos, itemsize=1) 252 if self.size < 1: 253 raise omero.ApiUsageException( 254 None, None, "String size must be > 0 (Column: %s)" % self.name) 255 return tables.StringCol(pos=pos, itemsize=self.size)
256
257 -class AbstractArrayColumn(AbstractColumn):
258 """ 259 Additional base logic for array columns 260 """ 261
262 - def __init__(self):
264
265 - def settable(self, tbl):
266 AbstractColumn.settable(self, tbl) 267 268 # Pytables 2.1 has the array size in Column.dtype.shape 269 #shape = getattr(tbl.cols, self.name).dtype.shape 270 #self.size = shape[0] 271 272 # Pytables 2.2 and later replaced this with Column.shape 273 #shape = getattr(tbl.cols, self.name).shape 274 #assert(len(shape) == 2) 275 #self.size = shape[1] 276 277 # http://www.pytables.org/trac-bck/ticket/231 278 # http://www.pytables.org/trac-bck/ticket/232 279 # TODO: Clean this up 280 281 # Taken from http://www.pytables.org/trac-bck/changeset/4176 282 column = getattr(tbl.cols, self.name) 283 self.size = column.descr._v_dtypes[column.name].shape[0]
284 285
286 - def arrays(self):
287 """ 288 Arrays of size 1 have to be converted to scalars, otherwise the 289 column-to-row conversion in HdfStorage.append() will fail. 290 This is messy, but I can't think of a better way. 291 """ 292 for v in self.values: 293 if len(v) != self.size: 294 raise omero.ValidationException( 295 None, None, "Column %s requires arrays of length %d" % 296 (self.name, self.size)) 297 298 if self.size == 1: 299 return [[v[0] for v in self.values]] 300 return [self.values]
301
302 - def dtypes(self):
303 """ 304 Overriding to correct for size. 305 """ 306 return [(self.name, self._types[0], self.size)]
307
308 -class FloatArrayColumnI(AbstractArrayColumn, omero.grid.FloatArrayColumn):
309
310 - def __init__(self, name = "Unknown", *args):
311 omero.grid.FloatArrayColumn.__init__(self, name, *args) 312 AbstractArrayColumn.__init__(self)
313
314 - def descriptor(self, pos):
315 # During initialization, size might be zero 316 if pos is None: 317 return tables.Float32Col(pos=pos) 318 if self.size < 1: 319 raise omero.ApiUsageException( 320 None, None, "Array length must be > 0 (Column: %s)" % self.name) 321 return tables.Float32Col(pos=pos, shape=self.size)
322
323 -class DoubleArrayColumnI(AbstractArrayColumn, omero.grid.DoubleArrayColumn):
324
325 - def __init__(self, name = "Unknown", *args):
326 omero.grid.DoubleArrayColumn.__init__(self, name, *args) 327 AbstractArrayColumn.__init__(self)
328
329 - def descriptor(self, pos):
330 # During initialization, size might be zero 331 if pos is None: 332 return tables.Float64Col(pos=pos) 333 if self.size < 1: 334 raise omero.ApiUsageException( 335 None, None, "Array length must be > 0 (Column: %s)" % self.name) 336 return tables.Float64Col(pos=pos, shape=self.size)
337
338 -class LongArrayColumnI(AbstractArrayColumn, omero.grid.LongArrayColumn):
339
340 - def __init__(self, name = "Unknown", *args):
341 omero.grid.LongArrayColumn.__init__(self, name, *args) 342 AbstractArrayColumn.__init__(self)
343
344 - def descriptor(self, pos):
345 # During initialization, size might be zero 346 if pos is None: 347 return tables.Int64Col(pos=pos) 348 if self.size < 1: 349 raise omero.ApiUsageException( 350 None, None, "Array length must be > 0 (Column: %s)" % self.name) 351 return tables.Int64Col(pos=pos, shape=self.size)
352
353 -class MaskColumnI(AbstractColumn, omero.grid.MaskColumn):
354
355 - def __init__(self, name = "Unknown", *args):
356 omero.grid.MaskColumn.__init__(self, name, *args) 357 AbstractColumn.__init__(self)
358
359 - def __noneorsame(self, a, b):
360 if a is None: 361 if b is None: 362 return 363 # a not none 364 if b is not None: 365 if len(a) == len(b): 366 return 367 raise omero.ValidationException(None, None, "Columns don't match")
368
369 - def __sanitycheck(self):
370 self.__noneorsame(self.imageId, self.theZ) 371 self.__noneorsame(self.imageId, self.theT) 372 self.__noneorsame(self.imageId, self.x) 373 self.__noneorsame(self.imageId, self.y) 374 self.__noneorsame(self.imageId, self.w) 375 self.__noneorsame(self.imageId, self.h) 376 self.__noneorsame(self.imageId, self.bytes)
377
378 - def descriptor(self, pos):
379 class MaskDescription(tables.IsDescription): 380 _v_pos = pos 381 i = tables.Int64Col(pos=0) 382 z = tables.Int32Col(pos=1) 383 t = tables.Int32Col(pos=2) 384 x = tables.Float64Col(pos=3) 385 y = tables.Float64Col(pos=4) 386 w = tables.Float64Col(pos=5) 387 h = tables.Float64Col(pos=6)
388 return MaskDescription()
389
390 - def arrays(self):
391 self.__sanitycheck() 392 a = [ 393 self.imageId, 394 self.theZ, 395 self.theT, 396 self.x, 397 self.y, 398 self.w, 399 self.h, 400 ] 401 return a
402
403 - def getsize(self):
404 self.__sanitycheck() 405 if self.imageId is None: 406 return None 407 else: 408 return len(self.imageId)
409
410 - def setsize(self, size):
411 if size is None: 412 self.imageId = None 413 self.theZ = None 414 self.theT = None 415 self.x = None 416 self.y = None 417 self.w = None 418 self.h = None 419 else: 420 dts = self.dtypes() 421 self.imageId = numpy.zeroes(size, dtype = dts[0]) 422 self.theZ = numpy.zeroes(size, dtype = dts[1]) 423 self.theT = numpy.zeroes(size, dtype = dts[2]) 424 self.x = numpy.zeroes(size, dtype = dts[3]) 425 self.y = numpy.zeroes(size, dtype = dts[4]) 426 self.w = numpy.zeroes(size, dtype = dts[5]) 427 self.h = numpy.zeroes(size, dtype = dts[6])
428
429 - def readCoordinates(self, tbl, rowNumbers):
430 self.__sanitycheck() 431 AbstractColumn.readCoordinates(self, tbl, rowNumbers) # calls fromrows 432 masks = self._getmasks(tbl) 433 if rowNumbers is None or len(rowNumbers) == 0: 434 rowNumbers = range(masks.nrows) 435 self.getbytes(masks, rowNumbers)
436
437 - def read(self, tbl, start, stop):
438 self.__sanitycheck() 439 AbstractColumn.read(self, tbl, start, stop) # calls fromrows 440 masks = self._getmasks(tbl) 441 rowNumbers = range(start, stop) 442 self.getbytes(masks, rowNumbers)
443
444 - def getbytes(self, masks, rowNumbers):
445 self.bytes = [] 446 for idx in rowNumbers: 447 self.bytes.append(masks[idx].tolist())
448
449 - def fromrows(self, all_rows):
450 rows = all_rows[self.name] 451 # WORKAROUND: http://www.zeroc.com/forums/bug-reports/4165-icepy-can-not-handle-buffers-longs-i64.html#post20468 452 self.imageId = rows["i"].tolist() 453 self.theZ = rows["z"].tolist() # ticket:1665 454 self.theT = rows["t"].tolist() # ticket:1665 455 self.x = rows["x"] 456 self.y = rows["y"] 457 self.w = rows["w"] 458 self.h = rows["h"]
459
460 - def append(self, tbl):
461 self.__sanitycheck() 462 masks = self._getmasks(tbl) 463 for x in self.bytes: 464 if isinstance(x, list): 465 # This occurs primarily in testing. 466 masks.append(numpy.array(x, dtype=tables.UInt8Atom())) 467 else: 468 masks.append(numpy.fromstring(x, count=len(x), dtype=tables.UInt8Atom()))
469
470 - def _getmasks(self, tbl):
471 n = tbl._v_name 472 f = tbl._v_file 473 p = tbl._v_parent 474 # http://www.zeroc.com/doc/Ice-3.3.1/manual/Slice.5.8.html#200 475 # Ice::Byte can be -128 to 127 OR 0 to 255, but using UInt8 for the moment 476 try: 477 masks = getattr(p, "%s_masks" % n) 478 except tables.NoSuchNodeError: 479 masks = f.createVLArray(p, "%s_masks" % n, tables.UInt8Atom()) 480 return masks
481 482 # Helpers 483 # ======================================================================== 484 485 # Conversion classes are for omero.model <--> ome.model only (no python) 486
487 -class ObjectFactory(Ice.ObjectFactory):
488
489 - def __init__(self, cls, f):
490 self.id = cls.ice_staticId() 491 self.f = f
492
493 - def create(self, string):
494 return self.f()
495
496 - def destroy(self):
497 pass
498
499 - def register(self, ic):
500 ic.addObjectFactory(self, self.id)
501 502 503 # Object factories 504 # ========================================================================= 505 506 ObjectFactories = { 507 FileColumnI: ObjectFactory(FileColumnI, lambda: FileColumnI()), 508 ImageColumnI: ObjectFactory(ImageColumnI, lambda: ImageColumnI()), 509 RoiColumnI: ObjectFactory(RoiColumnI, lambda: RoiColumnI()), 510 WellColumnI: ObjectFactory(WellColumnI, lambda: WellColumnI()), 511 PlateColumnI: ObjectFactory(PlateColumnI, lambda: PlateColumnI()), 512 BoolColumnI: ObjectFactory(BoolColumnI, lambda: BoolColumnI()), 513 DoubleColumnI: ObjectFactory(DoubleColumnI, lambda: DoubleColumnI()), 514 LongColumnI: ObjectFactory(LongColumnI, lambda: LongColumnI()), 515 StringColumnI: ObjectFactory(StringColumnI, lambda: StringColumnI()), 516 FloatArrayColumnI: ObjectFactory(FloatArrayColumnI, lambda: FloatArrayColumnI()), 517 DoubleArrayColumnI: ObjectFactory(DoubleArrayColumnI, lambda: DoubleArrayColumnI()), 518 LongArrayColumnI: ObjectFactory(LongArrayColumnI, lambda: LongArrayColumnI()), 519 MaskColumnI: ObjectFactory(MaskColumnI, lambda: MaskColumnI()) 520 } 521