)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"96fc700f40f31df89bf2104b3a58a703e1198398","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_49184dd2","line":9,"updated":"2019-03-10 10:06:33.000000000","message":"I don\u0027t understand how this patch can fix issue described in this bug.\nProblem here is that in e.g. test TestDvrRouter.test_dvr_ha_router_failover_with_gw it is expected that there will be one router created on 2 agents, so it will be \"master\" on one node and \"backup\" on second one.\nInstead of this we have 2 different routers created, like:\n\n(Pdb++) pp router1.ns_name\n\u0027qrouter-b62d6f9f-25e1-480c-8791-dab46d71ca84@agent1\u0027\n(Pdb++) pp router2.ns_name\n\u0027qrouter-f53d6dad-ed0f-43eb-83c0-ca1b50b89837@agent2\u0027\n\nRouters aren\u0027t connected to each other so that is IMO the issue here.","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"1e82cd55548f1f6437c253dac681913ea0666840","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_9e5ebbe7","line":9,"range":{"start_line":9,"start_character":12,"end_line":9,"end_character":20},"updated":"2019-03-19 15:46:00.000000000","message":"This bug may also be addressed by this fix:\nhttps://review.openstack.org/#/c/642295/","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"070cb83300acf66f812408a7502f80ca326124d4","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_6981898f","line":9,"in_reply_to":"5fc1f717_49184dd2","updated":"2019-03-10 11:44:20.000000000","message":"Neutron router uuid is not the critical data. That uuid has no effect to the failover case. For failover, the VRRP protocol vr_id [1] is the critical data. The code now gives all routers \"1\" for VRRP vr_id. The same VRRP vr_id, from the perspective of protocol, it will be treated as same router. Then two concurrent cases may have issue on VRRP master selection. Please my comment here [2].\n[1] https://tools.ietf.org/html/rfc5798#section-5.2.3\n[2] https://bugs.launchpad.net/neutron/+bug/1819160/comments/3","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"251e6d474d1e7a571e4f3a8c5dce483c2e1a8ffc","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_ef41f90c","line":9,"in_reply_to":"5fc1f717_6981898f","updated":"2019-03-10 17:27:24.000000000","message":"But those are still 2 different routers, not one configured on 2 hosts.\nI checked now with Your patch, after short time it is like:\n\n    (Pdb++) pp router1.ha_state\n    \u0027master\u0027\n    (Pdb++) pp router2.ha_state\n    \u0027master\u0027\n    (Pdb++)\n\nbecause those routers don\u0027t have any connectivity between each other.\nI think that we need to modify those tests to reuse create_ha_routers() method from https://github.com/openstack/neutron/blob/master/neutron/tests/functional/agent/l3/framework.py#L592\nand also to use \"_get_master_and_slave_routers\" from https://github.com/openstack/neutron/blob/master/neutron/tests/functional/agent/l3/framework.py#L613 instead of assuming that router1 is always master and router2 is always backup","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"1b754ade203f77bdb86e73699af6e7fb5a6b9bdf","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_118b2c6e","line":9,"in_reply_to":"5fc1f717_7e229902","updated":"2019-03-12 11:26:49.000000000","message":"After some locally test, I findout the VRRP id will have no influence on the cases. Because they will create different test bridges. Eace case will create a master agent with br-1 and stand-by br-2, they were connected by a veth pair devices:\nhttps://github.com/openstack/neutron/blob/master/neutron/tests/functional/agent/l3/test_dvr_router.py#L1317-L1323\nSo it is OK to have same vrrp_id for every case. But for log distinguishability, I still think give each failover a different VRRP id is nececessary for locating the problem.\nLet\u0027s investigate more about why these case get failed.","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"8bad54d9de9d9ce5e195af767513774b756f4bef","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Give each HA failover case an independent vrrp_id"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes-Bug: #1819160"},{"line_number":10,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5fc1f717_7e229902","line":9,"in_reply_to":"5fc1f717_ef41f90c","updated":"2019-03-11 08:12:29.000000000","message":"Why they are not connected?\n\nAccording to this journal log:\nhttp://logs.openstack.org/20/642220/2/check/neutron-functional/4a83b93/logs/journal.log\n\nYou can use \u0027VR_10\u0027, \u0027VR_11\u0027, \u0027VR_12\u0027 to grep all this three cases.\nIt is indeed like the failover excepted procedure, something like this:\n(1) on that master agent1\nKeepalived_vrrp[7224]: VRRP_Instance(VR_11) Entering BACKUP STATE\nKeepalived_vrrp[7224]: VRRP_Instance(VR_11) Entering MASTER STATE\n(2) on that backup agent2\nKeepalived_vrrp[13313]: VRRP_Instance(VR_11) Entering BACKUP STATE\n(3) call that \u0027fail_ha_router\u0027 for master router1\nKeepalived_vrrp[7224]: VRRP_Instance(VR_11) Entering FAULT STATE\nKeepalived_vrrp[7224]: VRRP_Instance(VR_11) Now in FAULT state\n(4) router2 in backup agent2 now become the master\nKeepalived_vrrp[13313]: VRRP_Instance(VR_11) Transition to MASTER STATE\nKeepalived_vrrp[13313]: VRRP_Instance(VR_11) Entering MASTER STATE\n\nWhy these VR_10, VR_11, VR_12 failover log exists more than once,\nit is because these case were running in these classes:\nTestL3AgentFipQosExtensionDVR\nTestRouterGatewayIPQosAgentExtensionDVR\nTestL3AgentFipPortForwardingExtensionDVR\nTestDvrRouter\n\nOne more thing, if you grep \"(VR_1)\" in that log, you may find there are a lot of router state transform. This can absolutely influence these failover cases.","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"c1bd9e10818a6ae1920eb3ea59dce0f30c5612f1","unresolved":false,"context_lines":[{"line_number":20,"context_line":"    169.254.0.13/24"},{"line_number":21,"context_line":"    169.254.0.14/24"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Closes-Bug: #1819160"},{"line_number":24,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5fc1f717_88740f9e","line":23,"range":{"start_line":23,"start_character":0,"end_line":23,"end_character":10},"updated":"2019-04-02 23:01:00.000000000","message":"NIt: shouldn\u0027t this be the Related-bug tag? I just approved https://review.openstack.org/#/c/647784/, which also closes the same bug","commit_id":"8b1a7084a00a1764070213b93a1986d973132a6e"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"718c3cb50cb4d525d4c2ef456ab45e2dd39d6932","unresolved":false,"context_lines":[{"line_number":20,"context_line":"    169.254.0.13/24"},{"line_number":21,"context_line":"    169.254.0.14/24"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Closes-Bug: #1819160"},{"line_number":24,"context_line":"Change-Id: I1216d96af40449ec16a852cc1f6c4f15c85f4546"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5fc1f717_3893e05e","line":23,"range":{"start_line":23,"start_character":0,"end_line":23,"end_character":10},"in_reply_to":"5fc1f717_88740f9e","updated":"2019-04-03 14:14:49.000000000","message":"Three patches are facing to this bug:\nhttps://review.openstack.org/#/c/647784/\nhttps://review.openstack.org/#/c/645225/\nhttps://review.openstack.org/#/c/642220/ (this)","commit_id":"8b1a7084a00a1764070213b93a1986d973132a6e"}],"neutron/agent/linux/keepalived.py":[{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"0fc9bc864e14cf1fbe6d2bcecdbf84f9d58caa15","unresolved":false,"context_lines":[{"line_number":395,"context_line":""},{"line_number":396,"context_line":"    def _output_config_file(self):"},{"line_number":397,"context_line":"        config_str \u003d self.config.get_config_str()"},{"line_number":398,"context_line":"        LOG.debug(\"Router %s keepalived config: %s\","},{"line_number":399,"context_line":"                  self.resource_id, config_str)"},{"line_number":400,"context_line":"        config_path \u003d self.get_full_config_file_path(\u0027keepalived.conf\u0027)"},{"line_number":401,"context_line":"        file_utils.replace_file(config_path, config_str)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5fc1f717_b7e25f00","line":398,"range":{"start_line":398,"start_character":26,"end_line":398,"end_character":28},"updated":"2019-04-08 14:47:27.000000000","message":"should use %(resource_id)s or similar along with a dict()","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"}],"neutron/tests/functional/agent/l3/test_dvr_router.py":[{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"1b754ade203f77bdb86e73699af6e7fb5a6b9bdf","unresolved":false,"context_lines":[{"line_number":1507,"context_line":""},{"line_number":1508,"context_line":"    def _test_dvr_ha_router_failover(self, enable_gw, vrrp_id\u003dNone):"},{"line_number":1509,"context_line":"        self._setup_dvr_ha_agents()"},{"line_number":1510,"context_line":"        self._setup_dvr_ha_bridges()"},{"line_number":1511,"context_line":""},{"line_number":1512,"context_line":"        router1 \u003d self._create_dvr_ha_router("},{"line_number":1513,"context_line":"            self.agent, enable_gw\u003denable_gw, vrrp_id\u003dvrrp_id)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5fc1f717_91115c55","line":1510,"range":{"start_line":1510,"start_character":8,"end_line":1510,"end_character":36},"updated":"2019-03-12 11:26:49.000000000","message":"@Slawek Kaplonski\nThe following routers were connected by these bridges and veth devices.","commit_id":"081fa95a5c92c4c61c83563d58255f7dcf5ef22d"}]}