)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"feda56f003b9f0f04ad1bb674484a2de6fbdf0ff","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Put HA router state change event into queue"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"A race condition exists between l3 agents after a HA router delete RPC:"},{"line_number":10,"context_line":"Once a fanout HA router delete notification comes to the shedulered"},{"line_number":11,"context_line":"agents, the master router in one agent will delete all HA attributes"},{"line_number":12,"context_line":"and processes, such as keepalived VRRP, but because the backup router"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_dead2aea","line":9,"range":{"start_line":9,"start_character":48,"end_line":9,"end_character":49},"updated":"2016-10-10 07:58:23.000000000","message":"s/a/an/","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Put HA router state change event into queue"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"A race condition exists between l3 agents after a HA router delete RPC:"},{"line_number":10,"context_line":"Once a fanout HA router delete notification comes to the shedulered"},{"line_number":11,"context_line":"agents, the master router in one agent will delete all HA attributes"},{"line_number":12,"context_line":"and processes, such as keepalived VRRP, but because the backup router"},{"line_number":13,"context_line":"in another agent has not ever got/run the HA router delete procedure,"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_31064b20","line":10,"range":{"start_line":10,"start_character":57,"end_line":10,"end_character":67},"updated":"2016-10-09 15:18:01.000000000","message":"spelling: scheduled","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Put HA router state change event into queue"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"A race condition exists between l3 agents after a HA router delete RPC:"},{"line_number":10,"context_line":"Once a fanout HA router delete notification comes to the shedulered"},{"line_number":11,"context_line":"agents, the master router in one agent will delete all HA attributes"},{"line_number":12,"context_line":"and processes, such as keepalived VRRP, but because the backup router"},{"line_number":13,"context_line":"in another agent has not ever got/run the HA router delete procedure,"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_db80e18d","line":10,"range":{"start_line":10,"start_character":57,"end_line":10,"end_character":67},"in_reply_to":"1a95cdbc_31064b20","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"feda56f003b9f0f04ad1bb674484a2de6fbdf0ff","unresolved":false,"context_lines":[{"line_number":12,"context_line":"and processes, such as keepalived VRRP, but because the backup router"},{"line_number":13,"context_line":"in another agent has not ever got/run the HA router delete procedure,"},{"line_number":14,"context_line":"or anything that delay the backup to remove its VRRP process, then the"},{"line_number":15,"context_line":"VRRP procotol will set its router state to master."},{"line_number":16,"context_line":"Then HaRouter in backup l3 agent will run a series of operations, such"},{"line_number":17,"context_line":"as respawn the external processes, notify neutron server that here has"},{"line_number":18,"context_line":"a HA router with master state."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_1e5072d3","line":15,"range":{"start_line":15,"start_character":5,"end_line":15,"end_character":13},"updated":"2016-10-10 07:58:23.000000000","message":"s/procotol/protocol/","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"9d2bfe341bd1fff515669c3e30841fe6ac4bd5e4","unresolved":false,"context_lines":[{"line_number":12,"context_line":"and processes, such as keepalived VRRP, but because the backup router"},{"line_number":13,"context_line":"in another agent has not ever got/run the HA router delete procedure,"},{"line_number":14,"context_line":"or anything that delay the backup to remove its VRRP process, then the"},{"line_number":15,"context_line":"VRRP procotol will set its router state to master."},{"line_number":16,"context_line":"Then HaRouter in backup l3 agent will run a series of operations, such"},{"line_number":17,"context_line":"as respawn the external processes, notify neutron server that here has"},{"line_number":18,"context_line":"a HA router with master state."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_1ec31258","line":15,"range":{"start_line":15,"start_character":5,"end_line":15,"end_character":13},"in_reply_to":"1a95cdbc_1e5072d3","updated":"2016-10-10 08:14:53.000000000","message":"Done, thank you.","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"feda56f003b9f0f04ad1bb674484a2de6fbdf0ff","unresolved":false,"context_lines":[{"line_number":15,"context_line":"VRRP procotol will set its router state to master."},{"line_number":16,"context_line":"Then HaRouter in backup l3 agent will run a series of operations, such"},{"line_number":17,"context_line":"as respawn the external processes, notify neutron server that here has"},{"line_number":18,"context_line":"a HA router with master state."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This may be racy when delete rpc try to remove HA router stuffs"},{"line_number":21,"context_line":"and enqueue_state_change try to add some resources."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_1e6792b7","line":18,"range":{"start_line":18,"start_character":0,"end_line":18,"end_character":1},"updated":"2016-10-10 07:58:23.000000000","message":"s/a/an/","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":17,"context_line":"as respawn the external processes, notify neutron server that here has"},{"line_number":18,"context_line":"a HA router with master state."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This may be racy when delete rpc try to remove HA router stuffs"},{"line_number":21,"context_line":"and enqueue_state_change try to add some resources."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"For the spawned processes, it will stay alive after HA router"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_f55cce1a","line":20,"range":{"start_line":20,"start_character":57,"end_line":20,"end_character":63},"updated":"2016-10-09 15:18:01.000000000","message":"s/stuffs/resources/","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":17,"context_line":"as respawn the external processes, notify neutron server that here has"},{"line_number":18,"context_line":"a HA router with master state."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This may be racy when delete rpc try to remove HA router stuffs"},{"line_number":21,"context_line":"and enqueue_state_change try to add some resources."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"For the spawned processes, it will stay alive after HA router"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"1a95cdbc_fb7bddbb","line":20,"range":{"start_line":20,"start_character":57,"end_line":20,"end_character":63},"in_reply_to":"1a95cdbc_f55cce1a","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"}],"neutron/agent/l3/agent.py":[{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":246,"context_line":"            self.driver,"},{"line_number":247,"context_line":"            self.metadata_driver)"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"        self._queue \u003d queue.RouterProcessingQueue()"},{"line_number":250,"context_line":"        super(L3NATAgent, self).__init__(host\u003dself.conf.host)"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"        self.target_ex_net_id \u003d None"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_55082228","side":"PARENT","line":249,"updated":"2016-10-09 15:18:01.000000000","message":"Why did this move to the HA\u0027s AgentMixin?","commit_id":"fcd47cca6e885df62f493348a9b9c3d9db44bf35"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"89364a8f99e55687b003f695ea56e3cf4be28bc7","unresolved":false,"context_lines":[{"line_number":246,"context_line":"            self.driver,"},{"line_number":247,"context_line":"            self.metadata_driver)"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"        self._queue \u003d queue.RouterProcessingQueue()"},{"line_number":250,"context_line":"        super(L3NATAgent, self).__init__(host\u003dself.conf.host)"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"        self.target_ex_net_id \u003d None"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_aa765303","side":"PARENT","line":249,"in_reply_to":"1a95cdbc_2d2ee2fa","updated":"2016-10-10 11:26:41.000000000","message":"Done","commit_id":"fcd47cca6e885df62f493348a9b9c3d9db44bf35"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":246,"context_line":"            self.driver,"},{"line_number":247,"context_line":"            self.metadata_driver)"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"        self._queue \u003d queue.RouterProcessingQueue()"},{"line_number":250,"context_line":"        super(L3NATAgent, self).__init__(host\u003dself.conf.host)"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"        self.target_ex_net_id \u003d None"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_9b7ad9bb","side":"PARENT","line":249,"in_reply_to":"1a95cdbc_55082228","updated":"2016-10-10 01:37:06.000000000","message":"https://review.openstack.org/#/c/275614/12/neutron/agent/l3/ha.py@134","commit_id":"fcd47cca6e885df62f493348a9b9c3d9db44bf35"},{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"f40903e74afa2640ce71035d2478f26d1f6f818c","unresolved":false,"context_lines":[{"line_number":246,"context_line":"            self.driver,"},{"line_number":247,"context_line":"            self.metadata_driver)"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"        self._queue \u003d queue.RouterProcessingQueue()"},{"line_number":250,"context_line":"        super(L3NATAgent, self).__init__(host\u003dself.conf.host)"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"        self.target_ex_net_id \u003d None"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_2d2ee2fa","side":"PARENT","line":249,"in_reply_to":"1a95cdbc_9b7ad9bb","updated":"2016-10-10 09:55:33.000000000","message":"That\u0027s fine - since that AgentMixin is a mixin class, we don\u0027t expect every variable to exist there.\n\nImagine the case where DVR\u0027s AgentMixin will also need to access self._queue - will it be needed to define there as well?\n\nPlease define it here instead.","commit_id":"fcd47cca6e885df62f493348a9b9c3d9db44bf35"}],"neutron/agent/l3/ha.py":[{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"110e9fc9d08a8e16102e9909172bfebc59636c00","unresolved":false,"context_lines":[{"line_number":122,"context_line":"                return routers[0]"},{"line_number":123,"context_line":"        except Exception:"},{"line_number":124,"context_line":"            msg \u003d _LE(\"Failed to fetch router information for \u0027%s\u0027\")"},{"line_number":125,"context_line":"            LOG.exception(msg, router_id)"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"}],"source_content_type":"text/x-python","patch_set":9,"id":"3aaa91ec_049c2e5c","line":125,"updated":"2016-06-28 11:44:15.000000000","message":"how will an exception help here?","commit_id":"a0e597ff3f61cdc7247bd46704fab9e2fa2b3fe6"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"d44d0d75c5f61e50582b30037af1f6a7ad1d250d","unresolved":false,"context_lines":[{"line_number":122,"context_line":"                return routers[0]"},{"line_number":123,"context_line":"        except Exception:"},{"line_number":124,"context_line":"            msg \u003d _LE(\"Failed to fetch router information for \u0027%s\u0027\")"},{"line_number":125,"context_line":"            LOG.exception(msg, router_id)"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"}],"source_content_type":"text/x-python","patch_set":9,"id":"3aaa91ec_1ef635f9","line":125,"in_reply_to":"3aaa91ec_049c2e5c","updated":"2016-06-28 13:12:04.000000000","message":"This patch uses this RPC to check whether the HA router is still existed, here just catch and log all the RPC errors. Maybe a retry will be needed.","commit_id":"a0e597ff3f61cdc7247bd46704fab9e2fa2b3fe6"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"110e9fc9d08a8e16102e9909172bfebc59636c00","unresolved":false,"context_lines":[{"line_number":133,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":134,"context_line":"        if not router:"},{"line_number":135,"context_line":"            LOG.warning("},{"line_number":136,"context_line":"                _LW(\"Will not do master state change actions\""},{"line_number":137,"context_line":"                    \"for HA router %s, unable to ensure that \""},{"line_number":138,"context_line":"                    \"the HA router is still existed\"), router_id)"},{"line_number":139,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":9,"id":"3aaa91ec_4490f64e","line":136,"updated":"2016-06-28 11:44:15.000000000","message":"missing space at the end of the line","commit_id":"a0e597ff3f61cdc7247bd46704fab9e2fa2b3fe6"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"d44d0d75c5f61e50582b30037af1f6a7ad1d250d","unresolved":false,"context_lines":[{"line_number":133,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":134,"context_line":"        if not router:"},{"line_number":135,"context_line":"            LOG.warning("},{"line_number":136,"context_line":"                _LW(\"Will not do master state change actions\""},{"line_number":137,"context_line":"                    \"for HA router %s, unable to ensure that \""},{"line_number":138,"context_line":"                    \"the HA router is still existed\"), router_id)"},{"line_number":139,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":9,"id":"3aaa91ec_13b90e7a","line":136,"in_reply_to":"3aaa91ec_4490f64e","updated":"2016-06-28 13:12:04.000000000","message":"Done","commit_id":"a0e597ff3f61cdc7247bd46704fab9e2fa2b3fe6"},{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":121,"context_line":"            LOG.warning("},{"line_number":122,"context_line":"                _LW(\"Will not do master state change actions \""},{"line_number":123,"context_line":"                    \"for HA router %s, unable to ensure that \""},{"line_number":124,"context_line":"                    \"the HA router is still existed.\"), router_id)"},{"line_number":125,"context_line":"            return"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"        router[\u0027ha_router_state\u0027] \u003d state"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_15505a0a","line":124,"range":{"start_line":124,"start_character":35,"end_line":124,"end_character":51},"updated":"2016-10-09 15:18:01.000000000","message":"nit: \"still exists\"","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":121,"context_line":"            LOG.warning("},{"line_number":122,"context_line":"                _LW(\"Will not do master state change actions \""},{"line_number":123,"context_line":"                    \"for HA router %s, unable to ensure that \""},{"line_number":124,"context_line":"                    \"the HA router is still existed.\"), router_id)"},{"line_number":125,"context_line":"            return"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"        router[\u0027ha_router_state\u0027] \u003d state"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_3b50e531","line":124,"range":{"start_line":124,"start_character":35,"end_line":124,"end_character":51},"in_reply_to":"1a95cdbc_15505a0a","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            timestamp\u003dtimestamp)"},{"line_number":134,"context_line":"        self._queue.add(update)"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"    def _ha_router_state_change_acitons(self, router):"},{"line_number":137,"context_line":"        router_id \u003d router[\u0027id\u0027]"},{"line_number":138,"context_line":"        state \u003d router.get(\u0027ha_router_state\u0027, \u0027backup\u0027)"},{"line_number":139,"context_line":"        LOG.info(_LI(\u0027Router %(router_id)s transitioned to %(state)s\u0027),"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_b5116654","line":136,"range":{"start_line":136,"start_character":32,"end_line":136,"end_character":39},"updated":"2016-10-09 15:18:01.000000000","message":"spelling: actions","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            timestamp\u003dtimestamp)"},{"line_number":134,"context_line":"        self._queue.add(update)"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"    def _ha_router_state_change_acitons(self, router):"},{"line_number":137,"context_line":"        router_id \u003d router[\u0027id\u0027]"},{"line_number":138,"context_line":"        state \u003d router.get(\u0027ha_router_state\u0027, \u0027backup\u0027)"},{"line_number":139,"context_line":"        LOG.info(_LI(\u0027Router %(router_id)s transitioned to %(state)s\u0027),"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_5b43f16b","line":136,"range":{"start_line":136,"start_character":32,"end_line":136,"end_character":39},"in_reply_to":"1a95cdbc_b5116654","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":7787,"name":"Kevin Benton","email":"kevin@benton.pub","username":"blak111"},"change_message_id":"09ef5141515850f69bbec145f68e53d6fb257296","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":129,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"        if not router:"},{"line_number":132,"context_line":"            LOG.warning("}],"source_content_type":"text/x-python","patch_set":17,"id":"bff0334d_d10f952b","line":129,"updated":"2017-04-12 20:26:41.000000000","message":"This now introduces a server call every time there is a state change. Can\u0027t we just fix the race by ensuring the agent cleans up the metadata proxy process no matter what when deleting the router?","commit_id":"b54ad78c6eccac157eb3f57a9121f86d60dbc283"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"90b1e702daace1db4b735620ef05d2ea8c86404b","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":129,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"        if not router:"},{"line_number":132,"context_line":"            LOG.warning("}],"source_content_type":"text/x-python","patch_set":17,"id":"9ff52f3f_6ec21fec","line":129,"in_reply_to":"bff0334d_d10f952b","updated":"2017-04-13 02:36:17.000000000","message":"Yes, this introduces a get_routers rpc every time if state change.\nThe race condition happens between router deleting and state change in the agent where the router is backup state. This enqueue_state_change and router_process_queue are running parallelly, which means that `ensure agent clean up` and `enqueue_state_change` may still have an unexpected call order. In the comments [1], that step 5-13 could happen. Actually the enqueue_state_change will do a lot action to a router, so put it into the router_process_queue is in principle better.\n\n\n[1] https://review.openstack.org/#/c/266109/2//COMMIT_MSG@9","commit_id":"b54ad78c6eccac157eb3f57a9121f86d60dbc283"},{"author":{"_account_id":7787,"name":"Kevin Benton","email":"kevin@benton.pub","username":"blak111"},"change_message_id":"899e00de0b31de401c61be5609d0e52c8a4e9034","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":129,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"        if not router:"},{"line_number":132,"context_line":"            LOG.warning("}],"source_content_type":"text/x-python","patch_set":18,"id":"7ffa3b31_d6bac8dc","line":129,"updated":"2017-04-16 04:19:42.000000000","message":"So if the neutron server is unresponsive, failovers can no longer be processed by the l3 agent. was that the case before?","commit_id":"1b3e77778188fe81214402c46f81c966155be68d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"3206f054840ad4330deb10a48d760aaf02a3b5fa","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":129,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"        if not router:"},{"line_number":132,"context_line":"            LOG.warning("}],"source_content_type":"text/x-python","patch_set":18,"id":"7ffa3b31_f8e12ad2","line":129,"in_reply_to":"7ffa3b31_7d24ec0d","updated":"2017-04-17 05:59:48.000000000","message":"Because we could trust the neutron server only, it knows and holds everything. For L3 agent, the router_info cache pop action is rival to this `enqueue_state_change`. So it starts with `enqueue_`, just put it to a queue.\n\nAnd, because this involved the neutron server. If it was dead, all the HA router fail over actions will be blocked. : (","commit_id":"1b3e77778188fe81214402c46f81c966155be68d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"99d2fcb960e00c92dcc11a348d1c6551bb420ace","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def enqueue_state_change(self, router_id, state):"},{"line_number":128,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":129,"context_line":"        router \u003d self._get_router(router_id)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"        if not router:"},{"line_number":132,"context_line":"            LOG.warning("}],"source_content_type":"text/x-python","patch_set":18,"id":"7ffa3b31_7d24ec0d","line":129,"in_reply_to":"7ffa3b31_d6bac8dc","updated":"2017-04-17 05:52:04.000000000","message":"No, this RPC was added since this patch.","commit_id":"1b3e77778188fe81214402c46f81c966155be68d"}],"neutron/tests/functional/agent/l3/framework.py":[{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                                                dual_stack\u003ddual_stack,"},{"line_number":226,"context_line":"                                                v6_ext_gw_with_sub\u003d("},{"line_number":227,"context_line":"                                                    v6_ext_gw_with_sub))"},{"line_number":228,"context_line":"        self.mock_plugin_api.get_routers.return_value \u003d (router_info,)"},{"line_number":229,"context_line":"        router \u003d self.manage_router(self.agent, router_info)"},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"        # Add multiple-IPv6-prefix internal router port"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_757efe6a","line":228,"updated":"2016-10-09 15:18:01.000000000","message":"All the changes in this file are duplicated in test_dvr_router.py and test_ha_router.py. Is there a better way?\n\n(Example: make self.generate_router_info() automatrically mock the retval, move the \u0027while not\u0027 loop from below to a function).","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                                                dual_stack\u003ddual_stack,"},{"line_number":226,"context_line":"                                                v6_ext_gw_with_sub\u003d("},{"line_number":227,"context_line":"                                                    v6_ext_gw_with_sub))"},{"line_number":228,"context_line":"        self.mock_plugin_api.get_routers.return_value \u003d (router_info,)"},{"line_number":229,"context_line":"        router \u003d self.manage_router(self.agent, router_info)"},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"        # Add multiple-IPv6-prefix internal router port"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_1b1b2982","line":228,"in_reply_to":"1a95cdbc_757efe6a","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"}],"neutron/tests/unit/agent/l3/test_agent.py":[{"author":{"_account_id":12444,"name":"John Schwarz","email":"jschwarz@redhat.com","username":"jschwarz"},"change_message_id":"1ec27296d3dcbe9f1cf84496316d8ff0dba0bded","unresolved":false,"context_lines":[{"line_number":212,"context_line":"        agent._update_metadata_proxy \u003d mock.Mock()"},{"line_number":213,"context_line":"        agent.enqueue_state_change(router_id, \u0027master\u0027)"},{"line_number":214,"context_line":"        agent._process_router_update()"},{"line_number":215,"context_line":"        self.assertFalse(agent._update_metadata_proxy.call_count)"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"    def test_periodic_sync_routers_task_raise_exception(self):"},{"line_number":218,"context_line":"        agent \u003d l3_agent.L3NATAgent(HOSTNAME, self.conf)"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_d58fd272","line":215,"range":{"start_line":215,"start_character":8,"end_line":215,"end_character":65},"updated":"2016-10-09 15:18:01.000000000","message":"Can you add the opposite case, where a router is updated and the metadata proxy is indeed updated following the _process_router_update() call?","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"943bd58798ab35868a5961ecf549764f81e06d04","unresolved":false,"context_lines":[{"line_number":212,"context_line":"        agent._update_metadata_proxy \u003d mock.Mock()"},{"line_number":213,"context_line":"        agent.enqueue_state_change(router_id, \u0027master\u0027)"},{"line_number":214,"context_line":"        agent._process_router_update()"},{"line_number":215,"context_line":"        self.assertFalse(agent._update_metadata_proxy.call_count)"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"    def test_periodic_sync_routers_task_raise_exception(self):"},{"line_number":218,"context_line":"        agent \u003d l3_agent.L3NATAgent(HOSTNAME, self.conf)"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a95cdbc_fb291d9c","line":215,"range":{"start_line":215,"start_character":8,"end_line":215,"end_character":65},"in_reply_to":"1a95cdbc_d58fd272","updated":"2016-10-10 01:37:06.000000000","message":"Done","commit_id":"89400413d4682de6ae4250a59ed8256a73446000"}]}