)]}'
{"octavia/controller/healthmanager/health_manager.py":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"85b7666195cc6aa1c83a741beee2b15f73db3b62","unresolved":false,"context_lines":[{"line_number":111,"context_line":"                amp_health \u003d None"},{"line_number":112,"context_line":"            except db_exc.DBConnectionError:"},{"line_number":113,"context_line":"                db_api.wait_for_connection(self.dead)"},{"line_number":114,"context_line":"                lock_session.rollback()"},{"line_number":115,"context_line":"                amp_health \u003d None"},{"line_number":116,"context_line":"                if not self.dead.is_set():"},{"line_number":117,"context_line":"                    time.sleep(CONF.health_manager.heartbeat_timeout)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_1a56765d","line":114,"updated":"2018-09-07 22:45:01.000000000","message":"Interesting that this doesn\u0027t raise an exception as at this point the session doesn\u0027t exit. It\u0027s good to have it in case it was just a network fracture and the session/lock is still present.","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"85b7666195cc6aa1c83a741beee2b15f73db3b62","unresolved":false,"context_lines":[{"line_number":113,"context_line":"                db_api.wait_for_connection(self.dead)"},{"line_number":114,"context_line":"                lock_session.rollback()"},{"line_number":115,"context_line":"                amp_health \u003d None"},{"line_number":116,"context_line":"                if not self.dead.is_set():"},{"line_number":117,"context_line":"                    time.sleep(CONF.health_manager.heartbeat_timeout)"},{"line_number":118,"context_line":"            except Exception:"},{"line_number":119,"context_line":"                with excutils.save_and_reraise_exception():"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_9a7926fd","line":116,"updated":"2018-09-07 22:45:01.000000000","message":"Probably should add a comment about why we are sleeping here.","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":6469,"name":"Carlos Gonçalves","display_name":"Carlos Goncalves","email":"cgoncalves@redhat.com","username":"cgoncalves"},"change_message_id":"c73dfe865cee822f3acc501fc8131af8f6f5f31f","unresolved":false,"context_lines":[{"line_number":113,"context_line":"                db_api.wait_for_connection(self.dead)"},{"line_number":114,"context_line":"                lock_session.rollback()"},{"line_number":115,"context_line":"                amp_health \u003d None"},{"line_number":116,"context_line":"                if not self.dead.is_set():"},{"line_number":117,"context_line":"                    time.sleep(CONF.health_manager.heartbeat_timeout)"},{"line_number":118,"context_line":"            except Exception:"},{"line_number":119,"context_line":"                with excutils.save_and_reraise_exception():"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_eda3c699","line":116,"in_reply_to":"3f79a3b5_9a7926fd","updated":"2018-09-08 00:22:35.000000000","message":"Done","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"}],"octavia/db/api.py":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"85b7666195cc6aa1c83a741beee2b15f73db3b62","unresolved":false,"context_lines":[{"line_number":67,"context_line":"            get_engine().scalar(select([1]))"},{"line_number":68,"context_line":"            down \u003d False"},{"line_number":69,"context_line":"            LOG.debug(\u0027Connection to database reestablished.\u0027)"},{"line_number":70,"context_line":"        except db_exc.DBConnectionError:"},{"line_number":71,"context_line":"            LOG.warning(\u0027Connection to database failed.\u0027)"},{"line_number":72,"context_line":"            time.sleep(cfg.CONF.database.retry_interval)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_fa3ffa1e","line":70,"updated":"2018-09-07 22:45:01.000000000","message":"Do we need to be this specific, or should we catch \"Exception\" here and retry on any error attempting this simple health check?\nThe outer check needs to be DBConnectionError for sure, just not sure if we should be more broad here.\nLike if the DB is missing after the restart for some transient reason, it would raise DBNonExistentDatabase","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"7e1bc76f36136e6c1cdd8d9a33bd8ab25a013d3b","unresolved":false,"context_lines":[{"line_number":67,"context_line":"            get_engine().scalar(select([1]))"},{"line_number":68,"context_line":"            down \u003d False"},{"line_number":69,"context_line":"            LOG.debug(\u0027Connection to database reestablished.\u0027)"},{"line_number":70,"context_line":"        except db_exc.DBConnectionError:"},{"line_number":71,"context_line":"            LOG.warning(\u0027Connection to database failed.\u0027)"},{"line_number":72,"context_line":"            time.sleep(cfg.CONF.database.retry_interval)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_10af35e3","line":70,"in_reply_to":"3f79a3b5_2dd69ec4","updated":"2018-09-08 00:34:53.000000000","message":"Right, because you moved the connect into the try block. My question is, when you did this test, did you wait the full interval? It seems like if the interval passed, and the DB restored, it would failover.","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":6469,"name":"Carlos Gonçalves","display_name":"Carlos Goncalves","email":"cgoncalves@redhat.com","username":"cgoncalves"},"change_message_id":"c73dfe865cee822f3acc501fc8131af8f6f5f31f","unresolved":false,"context_lines":[{"line_number":67,"context_line":"            get_engine().scalar(select([1]))"},{"line_number":68,"context_line":"            down \u003d False"},{"line_number":69,"context_line":"            LOG.debug(\u0027Connection to database reestablished.\u0027)"},{"line_number":70,"context_line":"        except db_exc.DBConnectionError:"},{"line_number":71,"context_line":"            LOG.warning(\u0027Connection to database failed.\u0027)"},{"line_number":72,"context_line":"            time.sleep(cfg.CONF.database.retry_interval)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_2dd69ec4","line":70,"in_reply_to":"3f79a3b5_fa3ffa1e","updated":"2018-09-08 00:22:35.000000000","message":"Good point. I\u0027d need to test breaking things at different places.\n\nOne quick test I did was renaming the database:\n\nhealth_check:\n\nSep 07 23:53:03 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: ERROR futurist.periodics [-] Failed to call periodic \u0027octavia.cmd.health_manager.periodic_health_check\u0027 (it runs every 3.00 seconds): ProgrammingError: (pymysql.err.ProgrammingError) (1146, u\"Table \u0027octavia.amphora_health\u0027 doesn\u0027t exist\") [SQL: u\u0027SELECT amphora_health.amphora_id AS amphora_health_amphora_id, amphora_health.last_update AS amphora_health_last_update, amphora_health.busy AS amphora_health_busy \\nFROM amphora_health \\nWHERE amphora_health.busy \u003d false AND amphora_health.last_update \u003c %(last_update_1)s \\n LIMIT %(param_1)s FOR UPDATE\u0027] [parameters: {u\u0027param_1\u0027: 1, u\u0027last_update_1\u0027: datetime.datetime(2018, 9, 7, 23, 52, 3, 358204)}] (Background on this error at: http://sqlalche.me/e/f405)\n[...]\n\n\nupdate_db: \n\nSep 07 23:53:04 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: ERROR octavia.controller.healthmanager.health_drivers.update_db [-] update_health encountered an unknown error processing health message \nfor amphora 2dcd6dbc-e755-4d6c-b0fa-2a4f1ba3ca74 with IP 192.168.0.6: ProgrammingError: (pymysql.err.ProgrammingError) (1146, u\"Table \u0027octavia.load_balancer\u0027 doesn\u0027t exist\") [SQL: u\u0027SELECT anon_1.load_balancer_c\nreated_at AS anon_1_load_balancer_created_at, [...]\n\n\nOnce I restored the database (renaming it back to \u0027octavia\u0027), things went back to normal. No failover was scheduled.\n\n\nSep 08 00:01:18 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027octavia.cmd.health_manager.periodic_health_check\u0027 {{(pid\u003d3040) _process_scheduled /usr/lib/python2.7/site-packages/futurist/periodics.py:639}}\nSep 08 00:01:21 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027octavia.cmd.health_manager.periodic_health_check\u0027 {{(pid\u003d3040) _process_scheduled /usr/lib/python2.7/site-packages/futurist/periodics.py:639}}\nSep 08 00:01:24 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027octavia.cmd.health_manager.periodic_health_check\u0027 {{(pid\u003d3040) _process_scheduled /usr/lib/python2.7/site-packages/futurist/periodics.py:639}}\nSep 08 00:01:24 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG octavia.amphorae.drivers.health.heartbeat_udp [-] Received packet from (\u0027192.168.0.6\u0027, 1052) {{(pid\u003d3039) dorecv /opt/stack/octavia/octavia/amphorae/drivers/health/heartbeat_udp.py:187}}\nSep 08 00:01:24 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG octavia.controller.healthmanager.health_drivers.update_db [-] Health Update finished in: 0.0720081329346 seconds {{(pid\u003d3052) _update_health /opt/stack/octavia/octavia/controller/healthmanager/health_drivers/update_db.py:271}}\nSep 08 00:01:27 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027octavia.cmd.health_manager.periodic_health_check\u0027 {{(pid\u003d3040) _process_scheduled /usr/lib/python2.7/site-packages/futurist/periodics.py:639}}\nSep 08 00:01:30 rdocloud-devstack3.rdocloud octavia-health-manager[3025]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027octavia.cmd.health_manager.periodic_health_check\u0027 {{(pid\u003d3040) _process_scheduled /usr/lib/python2.7/site-packages/futurist/periodics.py:639}}\nSep 08 00:01:30 rdocloud-devstack3.rdocloud octavia-housekeeping[398]: DEBUG octavia.cmd.house_keeping [-] Initiating spare amphora check... {{(pid\u003d398) spare_amphora_check /opt/stack/octavia/octavia/cmd/house_keeping.py:47}}\nSep 08 00:01:30 rdocloud-devstack3.rdocloud octavia-housekeeping[398]: DEBUG octavia.controller.housekeeping.house_keeping [-] Required Spare Amphora count : 0 {{(pid\u003d398) spare_check /opt/stack/octavia/octavia/controller/housekeeping/house_keeping.py:44}}\nSep 08 00:01:30 rdocloud-devstack3.rdocloud octavia-housekeeping[398]: DEBUG octavia.controller.housekeeping.house_keeping [-] Current Spare Amphora count : 0 {{(pid\u003d398) spare_check /opt/stack/octavia/octavia/controller/housekeeping/house_keeping.py:45}}\nSep 08 00:01:30 rdocloud-devstack3.rdocloud octavia-housekeeping[398]: DEBUG octavia.controller.housekeeping.house_keeping [-] Current spare amphora count satisfies the requirement {{(pid\u003d398) spare_check /opt/stack/octavia/octavia/controller/housekeeping/house_keeping.py:60}}","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"85b7666195cc6aa1c83a741beee2b15f73db3b62","unresolved":false,"context_lines":[{"line_number":68,"context_line":"            down \u003d False"},{"line_number":69,"context_line":"            LOG.debug(\u0027Connection to database reestablished.\u0027)"},{"line_number":70,"context_line":"        except db_exc.DBConnectionError:"},{"line_number":71,"context_line":"            LOG.warning(\u0027Connection to database failed.\u0027)"},{"line_number":72,"context_line":"            time.sleep(cfg.CONF.database.retry_interval)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_3a3a5233","line":71,"updated":"2018-09-07 22:45:01.000000000","message":"Maybe include that we are retrying?\nThis signals to the operator that we didn\u0027t abort an operation, but in fact are just continuing to retry to connect.","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":6469,"name":"Carlos Gonçalves","display_name":"Carlos Goncalves","email":"cgoncalves@redhat.com","username":"cgoncalves"},"change_message_id":"c73dfe865cee822f3acc501fc8131af8f6f5f31f","unresolved":false,"context_lines":[{"line_number":68,"context_line":"            down \u003d False"},{"line_number":69,"context_line":"            LOG.debug(\u0027Connection to database reestablished.\u0027)"},{"line_number":70,"context_line":"        except db_exc.DBConnectionError:"},{"line_number":71,"context_line":"            LOG.warning(\u0027Connection to database failed.\u0027)"},{"line_number":72,"context_line":"            time.sleep(cfg.CONF.database.retry_interval)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_3aad7230","line":71,"in_reply_to":"3f79a3b5_3a3a5233","updated":"2018-09-08 00:22:35.000000000","message":"Same thought crossed my mind, and looking at this again makes sense your proposal because the LOG print in L66 is a debug one, so wouldn\u0027t be seen unless debug\u003dTrue.","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"}],"releasenotes/notes/fix-health-check-db-outage-279b0bc1d0039312.yaml":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"85b7666195cc6aa1c83a741beee2b15f73db3b62","unresolved":false,"context_lines":[{"line_number":2,"context_line":"fixes:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Fixed an issue where Octavia was struggling with proper handling of DB"},{"line_number":5,"context_line":"    connectivity issues bringing down all running loadbalancers. The Health"},{"line_number":6,"context_line":"    Manager is more resilient to DB outages now."}],"source_content_type":"text/x-yaml","patch_set":1,"id":"3f79a3b5_1ad196fb","line":5,"updated":"2018-09-07 22:45:01.000000000","message":"Can we be more specific with this. I think this is confusing. Maybe \"When Octavia cannot reach the database (all database instances are down), \"","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"},{"author":{"_account_id":6469,"name":"Carlos Gonçalves","display_name":"Carlos Goncalves","email":"cgoncalves@redhat.com","username":"cgoncalves"},"change_message_id":"c73dfe865cee822f3acc501fc8131af8f6f5f31f","unresolved":false,"context_lines":[{"line_number":2,"context_line":"fixes:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Fixed an issue where Octavia was struggling with proper handling of DB"},{"line_number":5,"context_line":"    connectivity issues bringing down all running loadbalancers. The Health"},{"line_number":6,"context_line":"    Manager is more resilient to DB outages now."}],"source_content_type":"text/x-yaml","patch_set":1,"id":"3f79a3b5_300ef131","line":5,"in_reply_to":"3f79a3b5_1ad196fb","updated":"2018-09-08 00:22:35.000000000","message":"Done","commit_id":"e858497b206900a8519a7f9677337bf91c6dbd18"}]}