Package PyFoam :: Package Infrastructure :: Module FoamMetaServer
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Infrastructure.FoamMetaServer

  1  #  ICE Revision: $Id: /local/openfoam/Python/PyFoam/PyFoam/Infrastructure/FoamMetaServer.py 6441 2010-04-07T15:25:41.442177Z bgschaid  $  
  2  """A XMLRPC-Server that knows all PyFoam-Runs in its subnet""" 
  3   
  4  from ServerBase import ServerBase 
  5  import xmlrpclib,socket 
  6  from threading import Lock,Thread,Timer 
  7   
  8  from PyFoam.Infrastructure.Logging import foamLogger 
  9  from PyFoam.Infrastructure.NetworkHelpers import checkFoamServers 
 10  from PyFoam import configuration as config 
 11  from PyFoam.ThirdParty.IPy import IP 
 12   
 13  import sys,time,copy,os 
 14  from traceback import extract_tb 
 15   
 16  DO_WEBSYNC = config().getboolean('Metaserver','doWebsync') 
 17  WEBSERVER_RPCURL = "http://%(host)s/metaserver/xmlrpc/" % {"host":config().get('Metaserver','webhost')} 
 18  WEBSYNC_INTERVAL = config().getfloat('Metaserver','websyncInterval') 
 19   
20 -class FoamMetaServer(object):
21 """The Metaserver. 22 23 Collects all the known FoamServers. Then waits for the servers to 24 register themselves. Checks at regular intervalls whether the processes are still alive 25 """
26 - def __init__(self,port=None):
27 """@param port: The port on which the server should listen""" 28 if port==None: 29 port=config().getint("Metaserver","port") 30 31 foamLogger("server").info("Starting Server up") 32 self.pid=os.getpid() 33 try: 34 self.webserver = xmlrpclib.ServerProxy(WEBSERVER_RPCURL) 35 self.servers={} 36 self.dataLock=Lock() 37 self.startupLock=Lock() 38 39 self.collect() 40 41 self.checker=MetaChecker(self) 42 self.checker.setDaemon(True) 43 self.checker.start() 44 45 self._server=ServerBase(('',port),logRequests=False) 46 self._server.register_instance(self) 47 self._server.register_introspection_functions() 48 self._server.serve_forever() # occasional errors with "Broken pipe" 49 except KeyboardInterrupt: 50 foamLogger("server").warning("Keyboard interrupt") 51 except socket.error,reason: 52 foamLogger("server").error("Socket Error: "+str(reason)) 53 print "Can't start server, Problem with socket: ",reason[1] 54 except: 55 foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])) 56 foamLogger("server").error(str(sys.exc_info()[1])) 57 foamLogger("server").error("Traceback: "+str(extract_tb(sys.exc_info()[2])))
58
59 - def list(self):
60 """Returns a list of the found Foam-Runs""" 61 self.dataLock.acquire() 62 servers=copy.deepcopy(self.servers) 63 self.dataLock.release() 64 65 result={} 66 67 for idnum,info in servers.iteritems(): 68 result[idnum]=info._info 69 70 return result
71
72 - def collect(self):
73 """Starts a thread that collects the data of the servers from the net""" 74 collector=MetaCollector(self) 75 collector.setDaemon(True) 76 collector.start() 77 return True
78
79 - def scan(self,additional):
80 """Starts a thread that collects the data of the servers from the net 81 @param additional: a string with a list of additional subnets that should be scanned""" 82 collector=MetaCollector(self,additional=additional) 83 collector.setDaemon(True) 84 collector.start() 85 return True
86
87 - def kill(self):
88 """Exits the server""" 89 foamLogger("server").warning("Terminating due to request") 90 t=Timer(1.,self._suicide) 91 t.start() 92 return True
93
94 - def _suicide(self):
95 """The server kills itself""" 96 os.kill(self.pid,1)
97
98 - def registerServer(self,ip,pid,port,sync=True,external=False):
99 """Registers a new server via XMLRPC 100 @param ip: IP of the server 101 @param pid: Die PID at the server 102 @param port: the port at which the server is listening 103 @param sync: (optional) if to sync with the webserver or not 104 """ 105 return self._registerServer(ip,pid,port,sync=sync,external=True)
106
107 - def _registerServer(self,ip,pid,port,sync=True,external=False):
108 """Registers a new server 109 @param ip: IP of the server 110 @param pid: Die PID at the server 111 @param port: the port at which the server is listening 112 @param external: was called via XMLRPC 113 @param sync: (optional) if to sync with the webserver or not 114 """ 115 self.dataLock.acquire() 116 serverID="%s:%d" % (ip,port) 117 118 foamLogger("server").info("Registering: %s with PID: %d" % (serverID,pid)) 119 120 insertServer=False 121 try: 122 if self.servers.has_key(serverID): 123 # maybe it's another process 124 server=xmlrpclib.ServerProxy("http://%s:%d" % (ip,port)) 125 gotPid=server.pid() 126 if pid!=gotPid: 127 self.servers.pop(serverID) 128 foamLogger("server").warning("Server "+serverID+" changed PID from %d to %d" % (pid,gotPid)) 129 insertServer=True 130 else: 131 foamLogger("server").warning("Server "+serverID+" already registered") 132 else: 133 insertServer=True 134 135 if insertServer: 136 new=ServerInfo(ip,pid,port) 137 doIt=external 138 if not doIt: 139 doIt=new.checkValid() 140 141 if doIt: 142 new.queryData() # occasional errors with 'Connection refused' 143 self.servers[serverID]=new 144 foamLogger("server").debug("Inserted "+serverID) 145 except: 146 foamLogger("server").error("Registering Server "+serverID+" failed:"+str(sys.exc_info()[0])) 147 foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) 148 foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) 149 150 self.dataLock.release() 151 152 if DO_WEBSYNC and insertServer and sync: 153 foamLogger("server").info("Registering %s for webserver: %s" % (serverID,'new/%(ip)s/%(port)s/' % {'ip':ip, 'port':port})) 154 try: 155 self.webserver.new_process(ip, port) 156 except: 157 foamLogger("server").warning("Registering %s for webserver failed!" % (serverID)) 158 return True
159
160 - def deregisterServer(self,ip,pid,port,sync=True):
161 """Deregisters a server 162 @param ip: IP of the server 163 @param pid: Die PID at the server 164 @param port: the port at which the server is listening 165 @param sync: (optional) if to sync with the webserver or not 166 """ 167 self.dataLock.acquire() 168 serverID="%s:%d" % (ip,port) 169 foamLogger("server").info("Deregistering: %s with PID: %d" % (serverID,pid)) 170 171 try: 172 if self.servers.has_key(serverID): 173 self.servers.pop(serverID) 174 175 if DO_WEBSYNC and sync: 176 foamLogger("server").info("Deregistering %s from webserver: %s" % (serverID,'end/%(ip)s/%(port)s/' % {'ip':ip, 'port':port})) 177 try: 178 self.webserver.end_process(ip, port) 179 except: 180 foamLogger("server").warning("Deregistering %s from webserver failed" % (serverID)) 181 else: 182 foamLogger("server").warning("Server "+serverID+" not registered") 183 except: 184 foamLogger("server").error("Deregistering Server "+serverID+" failed:"+str(sys.exc_info()[0])) 185 foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) 186 foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) 187 188 self.dataLock.release() 189 190 return True
191
192 - def forwardCommand(self,ip,port,cmd):
193 """Forwards a RPC to another machine 194 @param ip: IP of the server 195 @param port: the port at which the server is listening 196 @param cmd: the command that should be executed there 197 @return: the result of the command 198 """ 199 result="" 200 try: 201 server=xmlrpclib.ServerProxy("http://%s:%d" % (ip,port)) 202 result=eval("server."+cmd) 203 foamLogger("server").debug("Forwarding to "+ip+"the command\""+cmd+"\" Result:"+str(result)) 204 except xmlrpclib.Fault,reason: 205 result="xmlrpclib.Fault: "+str(reason) 206 except socket.error,reason: 207 result="socket.error: "+str(reason) 208 except TypeError,reason: 209 result="Type error: ",reason 210 except SyntaxError,reason: 211 result="Syntax Error in:"+cmd 212 213 if result==None: 214 result="" 215 216 return result
217
218 -class ServerInfo(object):
219 """Contains the information about a server"""
220 - def __init__(self,ip,pid,port):
221 """ 222 @param ip: IP of the server 223 @param pid: Die PID at the server 224 @param port: the port at which the server is listening 225 """ 226 self._info={} 227 self._info["ip"]=ip 228 self._info["pid"]=pid 229 self._info["port"]=port
230
231 - def checkValid(self):
232 """Check with server whether this data item is still valid""" 233 result=False 234 235 foamLogger("server").debug("Checking "+self["ip"]+"@"+str(self["port"])) 236 237 try: 238 server=xmlrpclib.ServerProxy("http://%s:%d" % (self["ip"],self["port"])) 239 pid=server.pid() 240 if pid==self["pid"]: 241 result=True 242 except socket.timeout,reason: 243 foamLogger("server").info(self["ip"]+"@"+str(self["port"])+" seems to be dead") 244 except: 245 foamLogger("server").debug("Checking Valid "+self["ip"]+" failed:"+str(sys.exc_info()[0])) 246 foamLogger("server").debug("Reason:"+str(sys.exc_info()[1])) 247 foamLogger("server").debug("Trace:"+str(extract_tb(sys.exc_info()[2]))) 248 249 foamLogger("server").debug("Result for "+self["ip"]+"@"+str(self["port"])+" = "+str(result)) 250 251 return result
252
253 - def queryData(self):
254 """Ask the server for additional data""" 255 server=xmlrpclib.ServerProxy("http://%s:%d" % (self["ip"],self["port"])) 256 for name in ["commandLine","cwd","foamVersion","isParallel","mpi","pyFoamVersion","scriptName","user","hostname"]: 257 result=eval("server."+name+"()") 258 self[name]=result
259
260 - def __getitem__(self,key):
261 return self._info[key]
262
263 - def __setitem__(self,key,value):
264 self._info[key]=value
265
266 -class MetaChecker(Thread):
267 """Checks regularily whether the registered Servers are still alive"""
268 - def __init__(self,parent):
269 """@param parent: the FoamMetaServer that gets the information""" 270 Thread.__init__(self) 271 self.parent=parent 272 self.sleepTime=config().getfloat("Metaserver","checkerSleeping") 273 self.syncTimes={}
274
275 - def run(self):
276 foamLogger("server").info("Checker starting") 277 while True: 278 self.parent.startupLock.acquire() 279 foamLogger("server").debug("Start Checking") 280 281 self.parent.dataLock.acquire() 282 servers=copy.deepcopy(self.parent.servers) 283 self.parent.dataLock.release() 284 285 for key,obj in servers.iteritems(): 286 isOK=obj.checkValid() 287 if not isOK: 288 foamLogger("server").info("Server "+key+" not OK. Deregistering") 289 self.parent.deregisterServer(obj["ip"],obj["pid"],obj["port"]) 290 elif DO_WEBSYNC: 291 if not self.syncTimes.has_key(key): 292 self.syncTimes[key]=self.sleepTime 293 294 if self.syncTimes[key] >= WEBSYNC_INTERVAL: 295 self.syncTimes[key]=self.sleepTime 296 foamLogger("server").debug("Refreshing "+key+" on Webserver") 297 try: 298 self.parent.webserver.refresh(obj['ip'], obj['port']) 299 except socket.timeout, e: 300 pass 301 except: 302 foamLogger("server").warning("Unknown exception "+str(sys.exc_info()[0])+" while syncing with webserver %s" % (WEBSERVER_RPCURL)) 303 else: 304 self.syncTimes[key] += self.sleepTime 305 306 foamLogger("server").debug("Stop Checking - sleeping") 307 self.parent.startupLock.release() 308 time.sleep(self.sleepTime)
309
310 -class MetaCollector(Thread):
311 """Scans the net in a separate thread"""
312 - def __init__(self,parent,additional=None):
313 """@param parent: the FoamMetaServer that gets the information 314 @param additional: A string with alist of additional subnets that should be scanned""" 315 Thread.__init__(self) 316 self.parent=parent 317 self.additional=additional
318
319 - def run(self):
320 self.parent.startupLock.acquire() 321 foamLogger("server").info("Collector starting") 322 323 if DO_WEBSYNC: 324 foamLogger("server").info("Get Processes from Webserver") 325 try: 326 webserver = xmlrpclib.ServerProxy(WEBSERVER_RPCURL) 327 for ip,port,pid in webserver.running_processes(): 328 port = int(port) 329 try: 330 server=xmlrpclib.ServerProxy("http://%s:%d" % (ip,port)) 331 pid=server.pid() # occasional errors with 'Connection refused' 332 self.parent._registerServer(ip,pid,port,sync=False) 333 except: 334 foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])+" while registering %s:%s" % (ip, port)) 335 foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) 336 foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) 337 except: 338 foamLogger("server").warning("Unknown exception "+str(sys.exc_info()[0])+" while syncing with webserver %s" % (WEBSERVER_RPCURL)) 339 340 port=config().getint("Network","startServerPort") 341 length=config().getint("Network","nrServerPorts") 342 343 machines=config().get("Metaserver","searchservers") 344 345 addreses=machines.split(',') 346 if self.additional!=None: 347 addreses=self.additional.split(',')+addreses 348 349 for a in addreses: 350 foamLogger("server").info("Collecting in subnet "+a) 351 for host in IP(a): 352 try: 353 name,alias,rest =socket.gethostbyaddr(str(host)) 354 except socket.herror,reason: 355 # no name for the host 356 name="unknown" 357 358 foamLogger("server").debug("Collector Checking:"+str(host)+" "+name) 359 360 result=checkFoamServers(str(host),port,length) 361 if result!=None: 362 foamLogger("server").debug("Collector Found "+str(result)+" for "+name) 363 for p in result: 364 try: 365 server=xmlrpclib.ServerProxy("http://%s:%d" % (str(host),p)) 366 ip=server.ip() 367 pid=server.pid() 368 self.parent._registerServer(ip,pid,p) 369 except: 370 foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])+" while registering "+name) 371 foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) 372 foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) 373 else: 374 foamLogger("server").debug("Collector Found "+str(result)+" for "+name) 375 376 self.parent.startupLock.release() 377 378 foamLogger("server").info("Collector finished")
379