1
2
3
4
5
6
7
8 import re
9 import os
10 import Ice
11 import sys
12 import path
13 import time
14 import omero
15 import logging
16 import optparse
17 import fileinput
18 import exceptions
19
20 import omero.cli
21 import omero.util
22 import omero.util.temp_files
23 import omero_Constants_ice
24 import omero_ext.uuid as uuid
25
26 command_pattern = "^\s*(\w+)(\((.*)\))?(:(.*))?$"
27 command_pattern_compiled = re.compile(command_pattern)
28 log = logging.getLogger("omero.perf")
29
30 FILE_FORMAT = """
31 File format:
32 <blank> Ignored
33 # comment Ignored
34 ServerTime(repeat=100) Retrieve the server time 100 times
35 Import:<file> Import given file
36 Import(Screen:<id>):<file> Import given file into screen
37 Import(Dataset:<id>):<file> Import given file into project/dataset
38 Import(Project:<id>,Dataset:<id>):<file> Import given file into project/dataset
39 Import(Dataset:some name):<file> Import given file into a new dataset
40 Import(Dataset):<file> Import given file into last created dataset (or create a new one)
41
42 #
43 # "Import" is the name of a command available in the current context
44 # Use the "--list" command to print them all. All lines must be of the
45 # form: %s
46
47 """ % command_pattern
48
49
50
51
52
53
59
60
62 """
63 Single line-item in the configuration file
64 """
65
67 self.line = line.strip()
68 if not self.comment():
69 match = command_pattern_compiled.match(self.line)
70 if not match:
71 raise BadLine("Unexpected line: %s" % line)
72 self.command = match.group(1)
73 self.arguments = match.group(3)
74 self.path = match.group(5)
75 self.props = dict()
76
77 if self.arguments:
78 args = self.arguments.split(",")
79 for arg in args:
80 parts = arg.split("=", 1)
81 value = (len(parts) == 2 and parts[1] or "")
82 self.props[parts[0]] = value
83
84 log.debug("Found line: %s, %s, %s, %s", self.command, self.arguments, self.path, self.props)
85
87 return int(self.props.get("repeat","1"))
88
94
96 if self.comment():
97 return
98 m = getattr(self, "_op_%s" % self.command, None)
99 if m is None:
100 raise BadCommand("Unknown command: %s" % self.command)
101
102 m(ctx)
103
105 id = None
106 id_path = ctx.dir / ("%s.id" % name)
107 prop = self.props.get(name)
108
109 if prop is None:
110 return None
111
112 try:
113 id = int(prop)
114 log.debug("Using specified %s:%s" % (name, id))
115 except:
116
117 if prop == "":
118 try:
119 id = int(id_path.lines()[0])
120 except exceptions.Exception, e:
121 log.debug("No %s.id: %s", name, e)
122 prop = str(uuid.uuid4())
123
124 if id is not None:
125 log.debug("Using reload %s:%s" % (name, id))
126 else:
127 kls = getattr(omero.model, "%sI" % name)
128 obj = kls()
129 obj.name = omero.rtypes.rstring(prop)
130 obj = ctx.update_service().saveAndReturnObject(obj)
131 id = obj.id.val
132 log.debug("Created obj %s:%s" % (name, id))
133 id_path.write_text(str(id))
134 return id
135
137 link = ctx.query_service().findByQuery(\
138 "select link from %s link where link.parent.id = %s and link.child.id = %s"\
139 % (kls_name, parent.id.val, child.id.val), None)
140 if link:
141 log.debug("Found link %s:%s" % (kls_name, link.id.val))
142 else:
143 kls = getattr(omero.model, "%sI" % kls_name)
144 obj = kls()
145 obj.parent = parent
146 obj.child = child
147 obj = ctx.update_service().saveAndReturnObject(obj)
148 log.debug("Created link %s:%s" % (kls_name, obj.id.val))
149
151 p = path.path(self.path)
152 if not p.exists():
153 raise BadPath("File does not exist: %s" % self.path)
154
155 f = str(p.abspath())
156 out = ctx.dir / ("import_%s.out" % ctx.count)
157 err = ctx.dir / ("import_%s.err" % ctx.count)
158
159 args = ["import", "---file=%s" % str(out), "---errs=%s" % str(err), "-s", ctx.host(), "-k", ctx.key(), f]
160 s_id = self.create_obj(ctx, "Screen")
161 if s_id:
162 args.extend(["-r", str(s_id)])
163 p_id = self.create_obj(ctx, "Project")
164 d_id = self.create_obj(ctx, "Dataset")
165 if p_id and d_id:
166 self.create_link(ctx, "ProjectDatasetLink", omero.model.ProjectI(p_id, False), omero.model.DatasetI(d_id, False))
167 if d_id:
168 args.extend(["-d", str(d_id)])
169
170 ctx.cli.invoke(args)
171 if ctx.cli.rv != 0:
172 raise BadImport("Failed import: rv=%s" % ctx.cli.rv)
173 num_pix = len(out.lines())
174 log.debug("Import count: %s", num_pix)
175
178
181
182 -class Context(object):
183 """
184 Login context which can be used by any handler
185 for connecting to a single session.
186 """
187
188 - def __init__(self, id, reporter = None, client = None):
189 self.reporters = []
190 self.count = 0
191 self.id = id
192 if client is None:
193 self.client = omero.client(id)
194 self.client.setAgent("OMERO.perf_test")
195 self.client.createSession()
196 else:
197 self.client = client
198 self.services = {}
199 self.cli = omero.cli.CLI()
200 self.cli.loadplugins()
201 self.setup_dir()
202 log.debug("Running performance tests in %s", self.dir)
203
204 - def add_reporter(self, reporter):
205 self.reporters.append(reporter)
206
207 - def setup_dir(self):
208 self.dir = path.path(".") / ("perfdir-%s" % os.getpid())
209 if self.dir.exists():
210 raise exceptions.Exception("%s exists!" % self.dir)
211 self.dir.makedirs()
212
213
214 handler = logging.handlers.RotatingFileHandler(str(self.dir / "perf.log"), maxBytes = 10000000, backupCount = 5)
215 handler.setLevel(logging.DEBUG)
216 formatter = logging.Formatter(omero.util.LOGFORMAT)
217 handler.setFormatter(formatter)
218 logging.getLogger().addHandler(handler)
219
220 log.debug("Started: %s", time.ctime())
221
224
226 return self.client.getProperty("omero.host")
227
229 return self.client.sf.ice_getIdentity().name
230
231 - def report(self, command, start, stop, loops, rv):
232 for reporter in self.reporters:
233 reporter.report(command, start, stop, loops, rv)
234
235 - def _stateless(self, name, prx):
236 svc = self.services.get(name)
237 if svc:
238 return svc
239 else:
240 svc = self.client.sf.getByName(name)
241 svc = prx.uncheckedCast(svc)
242 self.services[name] = svc
243 return svc
244
245 - def query_service(self):
246 return self._stateless(omero.constants.QUERYSERVICE, omero.api.IQueryPrx)
247
248 - def config_service(self):
249 return self._stateless(omero.constants.CONFIGSERVICE, omero.api.IConfigPrx)
250
251 - def update_service(self):
252 return self._stateless(omero.constants.UPDATESERVICE, omero.api.IUpdatePrx)
253
254
256
259
261
262 (self.ctx.dir/"line.log").write_text(line, append = True)
263
264 item = Item(line)
265 if item.comment():
266 return
267
268 values = {}
269 total = 0.0
270 self.ctx.incr()
271 start = time.time()
272 loops = item.repeat()
273 for i in range(loops):
274 try:
275 rv = item.execute(self.ctx)
276 except ItemException, ie:
277 log.exception("Error")
278 sys.exit(1)
279 except exceptions.Exception, e:
280 log.debug("Error during execution: %s" % item.line.strip(), exc_info = True)
281 rv = e
282 errs = values.get("errs",0)
283 errs += 1
284 values["errs"] = errs
285
286 if loops > 1:
287 values["avg"] = total / loops
288
289 stop = time.time()
290 total += (stop - start)
291 self.ctx.report(item.command, start, stop, loops, values)
292
293
294
295
296
297
298
300 """
301 Abstract base class of all reporters
302 """
303
304 - def report(self, command, start, stop, loops, rv):
305 raise exceptions.Exception("Not implemented")
306
307
309
311 if dir is None:
312 self.stream = sys.stdout
313 else:
314 self.file = str(dir / "report.csv")
315 self.stream = open(self.file, "w")
316 print >>self.stream, "Command,Start,Stop,Elapsed,Average,Values"
317
318 - def report(self, command, start, stop, loops, values):
319 print >>self.stream, "%s,%s,%s,%s,%s,%s" % (command, start, stop, (stop-start), (stop-start)/loops, values)
320 self.stream.flush()
321
322
324
326 import tables
327 self.file = str(dir / "report.hdf")
328 self.hdf = tables.openFile(self.file, "w")
329 self.tbl = self.hdf.createTable("/", "report", {
330 "Command":tables.StringCol(pos=0, itemsize = 64),
331 "Start":tables.Float64Col(pos=1),
332 "Stop":tables.Float64Col(pos=2),
333 "Elapsed":tables.Float64Col(pos=3),
334 "Average":tables.Float64Col(pos=4),
335 "Values":tables.StringCol(pos=5, itemsize = 1000)
336 })
337 self.row = self.tbl.row
338
339 - def report(self, command, start, stop, loops, values):
348
349
351
353 return
354 import matplotlib.pyplot as plt
355 self.fig = plt.figure()
356 self.ax = fig.add_subplot(111)
357
358 - def report(self, command, start, stop, loops, values):
359 return
360 ax.set_ylim(-2,25)
361 ax.set_xlim(0, (last-first))
362 plt.show()
363
364
365
366
367
368
369
371 """
372 Primary method used by the command-line execution of
373 this module.
374 """
375
376 log.debug("Running perf on files: %s", files)
377 for file in files:
378 for line in file:
379 handler(line)
380 log.debug("Handled %s lines" % handler.ctx.count)
381