1
2 """
3 Application class that implements pyFoamComparator
4 """
5
6 import sys
7 import re
8 import string
9 from xml.dom.minidom import parse
10 import xml.dom
11 from os import path,environ
12 from optparse import OptionGroup
13
14 from PyFoam.Error import error
15 from PyFoam.Basics.Utilities import execute
16 from PyFoam.Execution.AnalyzedRunner import AnalyzedRunner
17 from PyFoam.Execution.ConvergenceRunner import ConvergenceRunner
18 from PyFoam.Execution.BasicRunner import BasicRunner
19 from PyFoam.Execution.UtilityRunner import UtilityRunner
20 from PyFoam.Execution.ParallelExecution import LAMMachine
21 from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile
22 from PyFoam.LogAnalysis.BoundingLogAnalyzer import BoundingLogAnalyzer
23 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
24 from PyFoam.Basics.CSVCollection import CSVCollection
25
26 from PyFoamApplication import PyFoamApplication
27 from PyFoam.FoamInformation import changeFoamVersion,injectVariables
28
29 from Decomposer import Decomposer
30
42
44 solver=OptionGroup(self.parser,
45 "Solver",
46 "Controls the behaviour of the solver")
47 self.parser.add_option_group(solver)
48 result=OptionGroup(self.parser,
49 "Results",
50 "What should be done with the solver results")
51 self.parser.add_option_group(solver)
52 behave=OptionGroup(self.parser,
53 "Behaviour",
54 "What should be done and output")
55 self.parser.add_option_group(behave)
56
57 behave.add_option("--test",
58 action="store_true",
59 default=False,
60 dest="test",
61 help="Only does the preparation steps but does not execute the actual solver an the end")
62
63 result.add_option("--removeOld",
64 action="store_true",
65 default=False,
66 dest="removeOld",
67 help="Remove the directories from an old run without asking")
68
69 result.add_option("--purge",
70 action="store_true",
71 default=False,
72 dest="purge",
73 help="Remove the case directories after evaluating")
74
75 result.add_option("--no-purge",
76 action="store_true",
77 default=False,
78 dest="nopurge",
79 help="Don't remove the case directories after evaluating")
80
81 solver.add_option("--steady",
82 action="store_true",
83 default=False,
84 dest="steady",
85 help="Only runs the solver until convergence")
86
87 behave.add_option("--showDictionary",
88 action="store_true",
89 default=False,
90 dest="showDict"
91 ,help="Shows the parameter-dictionary after the running of the solver")
92
93 solver.add_option("--no-server",
94 dest="server",
95 default=True,
96 action="store_false",
97 help="Don't start the process-control-server")
98
100 fName=self.parser.getArgs()[0]
101
102 dom=parse(fName)
103 doc=dom.documentElement
104
105 if doc.tagName!='comparator':
106 error("Wrong root-element",doc.tagName,"Expected: 'comparator'")
107
108 self.data=ComparatorData(doc)
109
110 purge=False
111 if doc.hasAttribute('purge'):
112 purge=eval(doc.getAttribute('purge'))
113 if self.opts.purge:
114 purge=self.opts.purge
115 if self.opts.nopurge:
116 purge=False
117
118 steady=False
119 if doc.hasAttribute('steady'):
120 steady=eval(doc.getAttribute('steady'))
121 if self.opts.steady:
122 purge=self.opts.steady
123
124 print " Parameters read OK "
125 print
126
127 aLog=open(self.data.id+".overview","w")
128 csv=CSVCollection(self.data.id+".csv")
129
130 rDir=self.data.id+".results"
131 execute("rm -rf "+rDir)
132 execute("mkdir "+rDir)
133
134 calculated=0
135 format="%%0%dd" % len(str(len(self.data)))
136
137 for i in range(len(self.data)):
138 runID=(format % i)
139 print >>aLog,runID,
140 csv["ID"]=runID
141
142 use,para=self.data[i]
143 para["template"]=self.data.template
144 para["extension"]=self.data.extension
145 para["id"]=self.data.id
146
147 if use:
148 calculated+=1
149
150 print "Executing Variation",i+1,"of",len(self.data),
151 if calculated!=i+1:
152 print "(",calculated,"actually calculated)"
153 else:
154 print
155
156 print "Parameters:",
157 for k,v in para.iteritems():
158 print "%s='%s' " % (k,v),
159 if v.find(" ")>=0 or v.find("\t")>=0:
160 v="'"+v+"'"
161 print >>aLog,v,
162 csv[k]=v
163
164 print
165
166 if not use:
167 print "Skipping because not all conditions are satisfied"
168 csv.clear()
169 print
170 continue
171
172 cName=("%s."+format) % (self.data.id, i)
173 log=open(cName+".log","w")
174
175 para["case"]=cName
176 print "Case-directory:",cName
177 para["results"]=path.join(rDir,runID)
178 print "Results directory:",para["results"]
179 execute("mkdir "+para["results"])
180
181 if path.exists(cName):
182 if self.opts.removeOld:
183 print " Removing old case-directory"
184 execute("rm -r "+cName)
185 else:
186 error("Case-directory",cName,"exists")
187
188 print " copying template"
189 out=execute("cp -r "+self.data.template+" "+cName)
190 print >>log,"---- Copying"
191 for l in out:
192 print >>log,l,
193
194 print " preparing"
195 ok,erg=self.data.prep.execute(para,log)
196 print >>aLog,ok,
197 csv["prepare OK"]=ok
198
199 for i in range(len(erg)):
200 print >>aLog,erg[i],
201 csv["Prepare %02d" % i]=erg[i]
202
203 aLog.flush()
204
205 if self.opts.test:
206 print " Skipping execution"
207 else:
208 print " running the solver"
209 sys.stdout.flush()
210
211 if steady:
212 runnerClass=ConvergenceRunner
213 else:
214 runnerClass=AnalyzedRunner
215
216 run=runnerClass(BoundingLogAnalyzer(doTimelines=True,progress=True),
217 argv=[self.data.solver,".",cName],
218 silent=True,
219 lam=Command.parallel,
220 server=self.opts.server)
221
222 run.start()
223 ok=run.runOK()
224 if ok:
225 print " executed OK"
226 else:
227 print " fatal error"
228
229 for aName in run.listAnalyzers():
230 a=run.getAnalyzer(aName)
231 if 'titles' in dir(a):
232 for tit in a.lines.getValueNames():
233 t,v=a.getTimeline(tit)
234 if len(v)>0:
235 para["result_"+aName+"_"+tit]=v[-1]
236
237 print >>aLog,run.runOK(),run.lastTime(),run.run.wallTime(),
238 csv["Run OK"]=run.runOK()
239 csv["End Time"]=run.lastTime()
240 csv["Wall Time"]=run.run.wallTime()
241 csv["Wall Time (Foam)"]=run.totalClockTime()
242 csv["CPU Time"]=run.totalCpuTime()
243 csv["Wall Time First Step"]=run.firstClockTime()
244 csv["CPU Time First Step"]=run.firstCpuTime()
245
246 para["endTime"]=run.lastTime()
247 para["runlog"]=run.logFile
248
249 if self.opts.showDict:
250 print para
251
252 print " evaluating results"
253
254 ok,erg=self.data.post.execute(para,log)
255
256 if Command.parallel!=None:
257 print " Stoping LAM"
258 Command.parallel.stop()
259 Command.parallel=None
260
261 if ok:
262 print " Evaluation OK",
263 else:
264 print " Evaluation failed",
265
266 if len(erg)>0:
267 print ":",erg,
268 print
269
270 print >>aLog,ok,
271 for i in range(len(erg)):
272 print >>aLog,erg[i],
273 csv["Post %02d" % i]=erg[i]
274
275 if purge:
276 print " removing the case-directory"
277 out=execute("rm -r "+cName)
278 print >>log,"---- Removing"
279 for l in out:
280 print >>log,l,
281
282 log.close()
283 print
284 print >>aLog
285 aLog.flush()
286 csv.write()
287
288 aLog.close()
289
291 """ The object that holds the actual data"""
292
294 """
295 @param doc: the parsed XML-data from which the object is constructed
296 """
297 self.name=doc.getAttribute("name")
298 if self.name=="":
299 error("No name for 'comparator' given")
300
301 base=doc.getElementsByTagName("base")
302 if base.length!=1:
303 error("One 'base'-element needed. Found",base.length)
304 self.__parseBase(base[0])
305
306 self.vList=[]
307 for v in doc.getElementsByTagName("variation"):
308 self.vList.append(Variation(v))
309
311 """@param e: The 'base'-element"""
312
313 self.template=path.expandvars(e.getAttribute("template"))
314 if self.template=="":
315 error("No template is given")
316 if not path.exists(self.template):
317 error("Template",self.template,"does not exist")
318 self.id=path.basename(self.template)
319 if e.hasAttribute('extension'):
320 self.extension=e.getAttribute('extension')
321 self.id+="."+self.extension
322 else:
323 self.extension=""
324 self.solver=e.getAttribute("solver")
325 if self.solver=="":
326 error("No solver is given")
327 prep=e.getElementsByTagName("preparation")
328 if prep.length!=1:
329 error("One 'preparation'-element needed. Found",prep.length)
330 self.prep=PreparationChain(prep[0])
331 post=e.getElementsByTagName("evaluation")
332 if post.length!=1:
333 error("One 'evaluation'-element needed. Found",post.length)
334 self.post=EvaluationChain(post[0])
335
337 """@return: The total number of variations"""
338 if len(self.vList)==0:
339 return 0
340 else:
341 nr=1l
342 for v in self.vList:
343 nr*=len(v)
344 return nr
345
347 """@param nr: Number of the variation
348 @return: dictionary with the variation"""
349 if nr>=len(self):
350 error("Index",nr,"of variation out of bounds: [0,",len(self)-1,"]")
351 result={}
352 tmp=nr
353 conditions=[]
354 for v in self.vList:
355 if (tmp % len(v))!=0:
356 conditions.append(v.condition)
357
358 k,val=v[tmp % len(v)]
359 result[k]=val
360 tmp/=len(v)
361
362 assert tmp==0
363
364 use=True
365 for c in conditions:
366 cond=replaceValues(c,result)
367 use=use and eval(cond)
368
369 return use,result
370
372 """Abstract base class for a number of commands"""
374 """@param c: XML-Subtree that represents the chain"""
375 self.commands=[]
376 for e in c.childNodes:
377 if e.nodeType!=xml.dom.Node.ELEMENT_NODE:
378 continue
379 if not e.tagName in self.table.keys():
380 error("Tagname",e.tagName,"not in table of valid tags",self.table.keys())
381 self.commands.append(self.table[e.tagName](e))
382
384 """Executes the chain
385 @param para:A dictionary with the parameters
386 @param log: Logfile to write to"""
387
388 result=[]
389 status=True
390 for c in self.commands:
391
392 if c.doIt(para):
393 ok,erg=c.execute(para,log)
394 else:
395 ok,erg=True,[]
396
397 status=ok and status
398 if erg!=None:
399 if type(erg)==list:
400 result+=erg
401 else:
402 result.append(erg)
403
404 return status,result
405
407 """Checks whether there is an object of a specific type"""
408
409 for o in self.commands:
410 if type(o)==typ:
411 return True
412
413 return False
414
416 """Chain of Preparation commands"""
418 self.table={"genericcommand":GenericCommand,
419 "derived":DerivedCommand,
420 "foamcommand":FoamCommand,
421 "foamutility":FoamUtilityCommand,
422 "initial":InitialCommand,
423 "dictwrite":DictWriteCommand,
424 "setdictionary":SetDictionaryCommand,
425 "decompose":DecomposeCommand,
426 "foamversion":FoamVersionCommand,
427 "changeenvironment":ChangeEnvironmentCommand,
428 "setenv":SetEnvironmentCommand,
429 "boundary":BoundaryCommand}
430 CommandChain.__init__(self,c)
431
433 """Chain of evaluation commands"""
435 self.table={"genericcommand":GenericCommand,
436 "derived":DerivedCommand,
437 "foamutility":FoamUtilityCommand,
438 "foamcommand":FoamCommand,
439 "dictionary":DictionaryCommand,
440 "reconstruct":ReconstructCommand,
441 "lastresult":LastResultCommand,
442 "changeenvironment":ChangeEnvironmentCommand,
443 "setenv":SetEnvironmentCommand,
444 "copylog":CopyLogCommand}
445 CommandChain.__init__(self,c)
446
448 result=e.getAttribute(name)
449 if result=="":
450 if default==None:
451 error("Missing attribute",name,"in element",e.tagName)
452 else:
453 return default
454 return result
455
457 """Replaces all strings enclosed by $$ with the parameters
458 @param orig: the original string
459 @param para: dictionary with the parameters"""
460
461 exp=re.compile("\$[^$]*\$")
462 tmp=orig
463
464 m=exp.search(tmp)
465 while m:
466 a,e=m.span()
467 pre=tmp[0:a]
468 post=tmp[e:]
469 mid=tmp[a+1:e-1]
470
471 if not mid in para.keys():
472 error("Key",mid,"not existing in keys",para.keys())
473
474 tmp=pre+para[mid]+post
475
476 m=exp.search(tmp)
477
478 return tmp
479
481
482 parallel=None
483
484 """Abstract base class of all commands"""
487
488 - def doIt(self,para):
492
494 """@param vals: Dictionary with the keywords
495 @return: A boolean whether it completed successfully and a list with results (None if no results are generated)"""
496 error("Execute not implemented for",type(self))
497
499 """Executes a shell command"""
503
505 cmd=replaceValues(self.command,para)
506 print " Executing ",cmd,
507 sys.stdout.flush()
508 out=execute(cmd)
509
510 if len(out)>0:
511 print " -->",len(out),"lines output"
512 for l in out:
513 print >>log,"---- Command:",cmd
514 print >>log,l,
515 else:
516 print
517
518 return True,None
519
521 """Derives an additional value"""
526
528 tmp=replaceValues(self.expression,para)
529 try:
530 val=eval(tmp)
531 except SyntaxError:
532 error("Syntax error in",tmp)
533 print " Setting",self.name,"to",val
534 para[self.name]=str(val)
535
536 return True,None
537
539 """Returns values from the chains dictionaries"""
543
545 if para.has_key(self.key):
546 return True,para[self.key]
547 else:
548 print "-----> ",self.key,"not in set of valid keys",para.keys()
549 print >>log,self.key,"not in set of valid keys of dictionary",para
550 return False,None
551
553 """Sets value in the chains dictionaries"""
558
560 para[self.key]=self.value
561 return True,None
562
564 """Changes the used OpenFOAM-version"""
568
574
576 """Sets an environment variable"""
581
583 val=replaceValues(self.val,para)
584 var=replaceValues(self.var,para)
585 print " Setting variable",var,"to",val
586 environ[var]=val
587
588 return True,None
589
591 """Changes Environment variables by executing a script-file"""
595
597 script=replaceValues(self.script,para)
598 print " Changing environment variables by executing",script
599 injectVariables(script)
600
601 return True,None
602
604 """Decomposes a case and generates the LAM"""
610
624
626 """Reconstructs a case and deleted the LAM"""
628 Command.__init__(self,c)
629 self.onlyLatest=False
630 if c.hasAttribute('onlyLatest'):
631 self.onlyLatest=eval(c.getAttribute('onlyLatest'))
632
647
649 """Executes a OpenFOAM-utility"""
654
667
669 """Executes a OpenFOAM-utility and extracts information"""
673
675 argv=[self.utility,".",para['case']]+self.options.split()
676 print " Executing and analyzing",string.join(argv),
677 sys.stdout.flush()
678 run=UtilityRunner(argv,silent=True,lam=Command.parallel,logname=string.join(argv,"_"))
679 run.add("data",self.regexp)
680 run.start()
681 data=run.analyzer.getData("data")
682 result=None
683 if data!=None:
684 result=[]
685 for a in data:
686 result.append(a)
687 if result==None:
688 print "no data",
689 else:
690 print result,
691
692 if run.runOK():
693 print
694 else:
695 print "---> there was a problem"
696
697 return run.runOK(),result
698
700 """Common class for commands that operate on dictionaries"""
704
706 f=replaceValues(self.filename,para)
707 v=replaceValues(self.value,para)
708 s=replaceValues(self.subexpression,para)
709 k=replaceValues(self.key,para)
710
711 try:
712 dictFile=ParsedParameterFile(f,backup=True)
713 val=dictFile[k]
714 except KeyError:
715 self.error("Key: ",k,"not existing in File",f)
716 except IOError,e:
717 self.error("Problem with file",k,":",e)
718
719 try:
720 exec "dictFile[k]"+s+"=v"
721 except Exception,e:
722 error("Problem with subexpression:",sys.exc_info()[0],":",e)
723
724 dictFile.writeFile()
725
726 return True,None
727
729 """Common class for commands that set values on fields"""
731 SetterCommand.__init__(self,c)
732 self.field=c.getAttribute("field")
733 self.filename=path.join("$case$","0",self.field)
734
736 """Sets an initial condition"""
741
745
747 """Sets a boundary condition"""
749 FieldSetterCommand.__init__(self,c)
750 self.patch=c.getAttribute("patch")
751 self.key="boundaryField"
752 self.subexpression="['"+self.patch+"']"
753 self.element=c.getAttribute("element")
754 if self.element=="":
755 self.element="value"
756 self.subexpression+="['"+self.element+"']"
757
761
763 """Writes a value to a dictionary"""
765 SetterCommand.__init__(self,c)
766 self.dir=c.getAttribute("directory")
767 self.dict=c.getAttribute("dictionary")
768 self.filename=path.join("$case$",self.dir,self.dict)
769 self.key=c.getAttribute("key")
770 self.subexpression=c.getAttribute("subexpression")
771 self.value=c.getAttribute("value")
772
776
778 """Copies the result of the last time-step to the resultsd directory"""
781
788
790 """Copies the log file to the results"""
793
795 print " Copy logfile"
796 execute("cp "+para["runlog"]+" "+para["results"])
797 return True,None
798
800 """Represents one variation"""
801
803 """@param e: the XML-data from which it is created"""
804
805 self.name=e.getAttribute("name")
806 if self.name=="":
807 error("No name for 'variation' given")
808 self.key=e.getAttribute("key")
809 if self.key=="":
810 error("No key for 'variation'",self.name," given")
811 self.condition=e.getAttribute("condition")
812 if self.condition=="":
813 self.condition="True"
814 self.values=[]
815 for v in e.getElementsByTagName("value"):
816 self.values.append(str(v.firstChild.data))
817
819 return "Variation "+self.name+" varies "+self.key+" over "+str(self.values)
820
822 """@return: number of values"""
823 return len(self.values)
824
826 return self.key,self.values[key]
827