summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBill Peck <bpeck@redhat.com>2011-10-27 10:14:38 -0400
committerDan Callaghan <dcallagh@redhat.com>2011-11-11 14:41:34 +1000
commitda9be21a97fd6217485cb1f906c0b3d5e0a48ad4 (patch)
treed499f2f74b341dc0dad7b205127a61d7030acc7e
parent4b64b12060841a78a561ac0843a99444beac7047 (diff)
try except handling in wrong place for beaker-watchdog
-rw-r--r--LabController/proxy/src/bkr/labcontroller/watchdog.py165
1 files changed, 86 insertions, 79 deletions
diff --git a/LabController/proxy/src/bkr/labcontroller/watchdog.py b/LabController/proxy/src/bkr/labcontroller/watchdog.py
index 9f171ba..05dd5f0 100644
--- a/LabController/proxy/src/bkr/labcontroller/watchdog.py
+++ b/LabController/proxy/src/bkr/labcontroller/watchdog.py
@@ -49,85 +49,92 @@ def main_loop(conf=None, foreground=False):
sys.stderr.write("Error initializing Watchdog: %s\n" % ex)
sys.exit(1)
- try:
- """
- As watchdog expire_recipe calls recipes.stop on the server
- side it needs to be auth'ed, and we don't yet have a good
- way to auth via MRG. So xmlrpc will still need to be used for
- that call, and thus we must login.
- """
- if conf['QPID_BUS']:
- watchdog.hub._login()
- auth_renew = RepeatTimer(conf['RENEW_SESSION_INTERVAL'],
- watchdog.hub._login, stop_on_exception=False)
- auth_renew.daemon = True
- auth_renew.start()
- lbb = LabBeakerBus(watchdog=watchdog)
- active_watchdogs = lbb.rpc.recipes.tasks.watchdogs('active', lbb.lc)
- watchdog.active_watchdogs(active_watchdogs, purge=False)
- listen_to = conf.get('QPID_LISTEN_TO', [])
- lbb.run(listen_to)
- while True:
- try:
- # To avoid a polling style scenario here we would need the harness to be updating
- # us when the monitered files change size
- if not watchdog.run():
- logger.debug(80 * '-')
- watchdog.sleep()
- except (ShutdownException, KeyboardInterrupt):
- # ignore keyboard interrupts and sigterm
- auth_renew.stop()
- signal.signal(signal.SIGINT, signal.SIG_IGN)
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
- logger.info('Exiting...')
- break
-
- else:
- time_of_last_check = 0
- while True:
- try:
- now = time.time()
- # Poll for watchdogs
- if now - time_of_last_check > 60:
- time_of_last_check = now
- watchdog.hub._login()
- try:
- active_watchdogs = watchdog.hub.recipes.tasks.watchdogs('active')
- except xmlrpclib.Fault:
- # catch any xmlrpc errors
- traceback = Traceback()
- logger.error(traceback.get_traceback())
- try:
- expired_watchdogs = watchdog.hub.recipes.tasks.watchdogs('expired')
- except xmlrpclib.Fault:
- # catch any xmlrpc errors
- traceback = Traceback()
- logger.error(traceback.get_traceback())
-
- watchdog.expire_watchdogs(expired_watchdogs)
- watchdog.active_watchdogs(active_watchdogs)
- if not watchdog.run():
- logger.debug(80 * '-')
- watchdog.sleep()
- # FIXME: Check for recipes that match systems under
- # this lab controller, if so take recipe and provision
- # system.
- # write to stdout / stderr
- sys.stdout.flush()
- sys.stderr.flush()
- except socket.sslerror:
- pass # try again later
- except (ShutdownException, KeyboardInterrupt):
- # ignore keyboard interrupts and sigterm
- signal.signal(signal.SIGINT, signal.SIG_IGN)
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
- logger.info('Exiting...')
- break
- except:
- # this is a little extreme: log the exception and continue
- traceback = Traceback()
- logger.error(traceback.get_traceback())
- watchdog.sleep()
+ """
+ As watchdog expire_recipe calls recipes.stop on the server
+ side it needs to be auth'ed, and we don't yet have a good
+ way to auth via MRG. So xmlrpc will still need to be used for
+ that call, and thus we must login.
+ """
+ if conf['QPID_BUS']:
+ watchdog.hub._login()
+ auth_renew = RepeatTimer(conf['RENEW_SESSION_INTERVAL'],
+ watchdog.hub._login, stop_on_exception=False)
+ auth_renew.daemon = True
+ auth_renew.start()
+ lbb = LabBeakerBus(watchdog=watchdog)
+ active_watchdogs = lbb.rpc.recipes.tasks.watchdogs('active', lbb.lc)
+ watchdog.active_watchdogs(active_watchdogs, purge=False)
+ listen_to = conf.get('QPID_LISTEN_TO', [])
+ lbb.run(listen_to)
+ while True:
+ try:
+ # To avoid a polling style scenario here we would need the harness to be updating
+ # us when the monitered files change size
+ if not watchdog.run():
+ logger.debug(80 * '-')
+ watchdog.sleep()
+ except socket.sslerror:
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+ except xmlrpclib.ProtocolError:
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+ except (ShutdownException, KeyboardInterrupt):
+ # ignore keyboard interrupts and sigterm
+ auth_renew.stop()
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ logger.info('Exiting...')
+ break
+
+ else:
+ time_of_last_check = 0
+ while True:
+ try:
+ now = time.time()
+ # Poll for watchdogs
+ if now - time_of_last_check > 60:
+ time_of_last_check = now
+ watchdog.hub._login()
+ try:
+ active_watchdogs = watchdog.hub.recipes.tasks.watchdogs('active')
+ except xmlrpclib.Fault:
+ # catch any xmlrpc errors
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+ active_watchdogs = []
+
+ try:
+ expired_watchdogs = watchdog.hub.recipes.tasks.watchdogs('expired')
+ except xmlrpclib.Fault:
+ # catch any xmlrpc errors
+ expired_watchdogs = []
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+
+ watchdog.expire_watchdogs(expired_watchdogs)
+ watchdog.active_watchdogs(active_watchdogs)
+ if not watchdog.run():
+ logger.debug(80 * '-')
+ watchdog.sleep()
+ # FIXME: Check for recipes that match systems under
+ # this lab controller, if so take recipe and provision
+ # system.
+ # write to stdout / stderr
+ sys.stdout.flush()
+ sys.stderr.flush()
+ except socket.sslerror:
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+ except xmlrpclib.ProtocolError:
+ traceback = Traceback()
+ logger.error(traceback.get_traceback())
+ except (ShutdownException, KeyboardInterrupt):
+ # ignore keyboard interrupts and sigterm
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ logger.info('Exiting...')
+ break
def main():
parser = OptionParser()