)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"5acb0c40210400a4d55be75a01add33b07ccd7f3","unresolved":false,"context_lines":[{"line_number":15,"context_line":"keepalived all the time. Restart l3-agent will not help."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"This patch will check and delete the orphan vrrp process"},{"line_number":18,"context_line":"if there is any, at the startup of l3-agent."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"More details can be found in the bug description and comments."},{"line_number":21,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9a8ffd7b_a849a132","line":18,"updated":"2015-11-30 08:26:17.000000000","message":"I don\u0027t think requiring l3 agent restart to make HA router back to life is the best approach we can have. Have you looked into extending ProcessMonitor to allow for custom actions on process death? If we could make it react on the death event with vrrp pid file cleanup, it would remove the requirement for agent restart (not a cheap operation in huge clouds).","commit_id":"ed8762ea237d372d5640a494688f8b4bf10247ac"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"2bedc5112bf9759b1aa47b4ecc608eba56aa8a85","unresolved":false,"context_lines":[{"line_number":15,"context_line":"keepalived all the time. Restart l3-agent will not help."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"This patch will check and delete the orphan vrrp process"},{"line_number":18,"context_line":"if there is any, at the startup of l3-agent."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"More details can be found in the bug description and comments."},{"line_number":21,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9a8ffd7b_d97dd56d","line":18,"in_reply_to":"9a8ffd7b_a849a132","updated":"2015-11-30 09:38:24.000000000","message":"@Ihar Hrachyshka, Thanks for the advice. I do think requiring restart is not the best approach. I had considered extending ProcessMonitor  to do the job here. But as you can see, it would be more complicating. I would like to treat this patch as the first step to resolve the problem. If community accept the approach here(killing orphan vrrp process from neutron code), I will create another patch to handle the problem without restarting. What do you think about it?","commit_id":"ed8762ea237d372d5640a494688f8b4bf10247ac"}],"neutron/agent/linux/keepalived.py":[{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"77a27e2445eaa79cc5861f2bc33e6cbce45500cc","unresolved":false,"context_lines":[{"line_number":437,"context_line":"            # If keepalived process crashed unexpectedly, the vrrp process"},{"line_number":438,"context_line":"            # will be orphan and prevent keepalived process to be spawned."},{"line_number":439,"context_line":"            # A check here will let the l3-agent to kill the orphan process"},{"line_number":440,"context_line":"            # and spawn keealived successfully."},{"line_number":441,"context_line":"            if vrrp_pm.active:"},{"line_number":442,"context_line":"                vrrp_pm.disable()"},{"line_number":443,"context_line":"            cmd \u003d [\u0027keepalived\u0027, \u0027-P\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_a5f41687","line":440,"updated":"2015-12-15 16:28:58.000000000","message":"keepalived","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"ce3962ba9ab3b2526954e019aff2a7416df67786","unresolved":false,"context_lines":[{"line_number":437,"context_line":"            # If keepalived process crashed unexpectedly, the vrrp process"},{"line_number":438,"context_line":"            # will be orphan and prevent keepalived process to be spawned."},{"line_number":439,"context_line":"            # A check here will let the l3-agent to kill the orphan process"},{"line_number":440,"context_line":"            # and spawn keealived successfully."},{"line_number":441,"context_line":"            if vrrp_pm.active:"},{"line_number":442,"context_line":"                vrrp_pm.disable()"},{"line_number":443,"context_line":"            cmd \u003d [\u0027keepalived\u0027, \u0027-P\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_50edd7ed","line":440,"in_reply_to":"5a710552_a5f41687","updated":"2015-12-16 03:35:30.000000000","message":"Done","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":8873,"name":"Assaf Muller","email":"amuller@redhat.com","username":"amuller"},"change_message_id":"0711a43eebd0c6541bb404494ff044e0c3831d41","unresolved":false,"context_lines":[{"line_number":405,"context_line":"        pm \u003d self.get_process()"},{"line_number":406,"context_line":"        pm.disable(sig\u003d\u002715\u0027)"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"    def get_process(self, callback\u003dNone):"},{"line_number":409,"context_line":"        return external_process.ProcessManager("},{"line_number":410,"context_line":"            cfg.CONF,"},{"line_number":411,"context_line":"            self.resource_id,"}],"source_content_type":"text/x-python","patch_set":4,"id":"3a7e1126_2d8c8d34","line":408,"updated":"2015-12-17 14:17:57.000000000","message":"The \u0027callback\u0027 parameter is now unused as a result of this patch, can you please remove it in a follow up?","commit_id":"08171f58e9073cc071151f265a25174fb0299aa6"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"6cde153f8cc4ba955bd25f6f5599f012d7ac13ff","unresolved":false,"context_lines":[{"line_number":405,"context_line":"        pm \u003d self.get_process()"},{"line_number":406,"context_line":"        pm.disable(sig\u003d\u002715\u0027)"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"    def get_process(self, callback\u003dNone):"},{"line_number":409,"context_line":"        return external_process.ProcessManager("},{"line_number":410,"context_line":"            cfg.CONF,"},{"line_number":411,"context_line":"            self.resource_id,"}],"source_content_type":"text/x-python","patch_set":4,"id":"3a7e1126_19ef8f88","line":408,"in_reply_to":"3a7e1126_2d8c8d34","updated":"2015-12-17 14:56:46.000000000","message":"Will do it in a following patch.","commit_id":"08171f58e9073cc071151f265a25174fb0299aa6"}],"neutron/tests/functional/agent/linux/test_keepalived.py":[{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"77a27e2445eaa79cc5861f2bc33e6cbce45500cc","unresolved":false,"context_lines":[{"line_number":54,"context_line":"        self.assertEqual(self.expected_config.get_config_str(),"},{"line_number":55,"context_line":"                         self.manager.get_conf_on_disk())"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"    def test_keepalived_respawns(self):"},{"line_number":58,"context_line":"        self.manager.spawn()"},{"line_number":59,"context_line":"        process \u003d self.manager.get_process()"},{"line_number":60,"context_line":"        pid \u003d process.pid"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_a52176b6","line":57,"updated":"2015-12-15 16:28:58.000000000","message":"The test skipped now so please rebase and re-enable it in this patch: https://review.openstack.org/#/c/251693/","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"ce3962ba9ab3b2526954e019aff2a7416df67786","unresolved":false,"context_lines":[{"line_number":54,"context_line":"        self.assertEqual(self.expected_config.get_config_str(),"},{"line_number":55,"context_line":"                         self.manager.get_conf_on_disk())"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"    def test_keepalived_respawns(self):"},{"line_number":58,"context_line":"        self.manager.spawn()"},{"line_number":59,"context_line":"        process \u003d self.manager.get_process()"},{"line_number":60,"context_line":"        pid \u003d process.pid"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_b0b43bf2","line":57,"in_reply_to":"5a710552_a52176b6","updated":"2015-12-16 03:35:30.000000000","message":"Thanks for the reminder, and done","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":8873,"name":"Assaf Muller","email":"amuller@redhat.com","username":"amuller"},"change_message_id":"4b1a6ba2ff962929e747ade9953a2a4bafc7ae81","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"        # Exit the process, and see that when it comes back"},{"line_number":68,"context_line":"        # It\u0027s indeed a different process"},{"line_number":69,"context_line":"        utils.execute([\u0027kill\u0027, \u0027-15\u0027, pid], run_as_root\u003dTrue)"},{"line_number":70,"context_line":"        utils.wait_until_true("},{"line_number":71,"context_line":"            lambda: process.active and pid !\u003d process.pid,"},{"line_number":72,"context_line":"            timeout\u003d5,"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_581c72c1","line":69,"updated":"2015-12-09 13:58:30.000000000","message":"I believe that with signal 15 the test will pass with out the changes to keepalived.py. Shouldn\u0027t this stay signal 9 to make sure the changes actually made a difference?","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"ce3962ba9ab3b2526954e019aff2a7416df67786","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"        # Exit the process, and see that when it comes back"},{"line_number":68,"context_line":"        # It\u0027s indeed a different process"},{"line_number":69,"context_line":"        utils.execute([\u0027kill\u0027, \u0027-15\u0027, pid], run_as_root\u003dTrue)"},{"line_number":70,"context_line":"        utils.wait_until_true("},{"line_number":71,"context_line":"            lambda: process.active and pid !\u003d process.pid,"},{"line_number":72,"context_line":"            timeout\u003d5,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_d0b787f3","line":69,"in_reply_to":"5a710552_c5dda243","updated":"2015-12-16 03:35:30.000000000","message":"Done","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"77a27e2445eaa79cc5861f2bc33e6cbce45500cc","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"        # Exit the process, and see that when it comes back"},{"line_number":68,"context_line":"        # It\u0027s indeed a different process"},{"line_number":69,"context_line":"        utils.execute([\u0027kill\u0027, \u0027-15\u0027, pid], run_as_root\u003dTrue)"},{"line_number":70,"context_line":"        utils.wait_until_true("},{"line_number":71,"context_line":"            lambda: process.active and pid !\u003d process.pid,"},{"line_number":72,"context_line":"            timeout\u003d5,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a710552_c5dda243","line":69,"in_reply_to":"7a740942_2e9efc63","updated":"2015-12-15 16:28:58.000000000","message":"For the race condition, I don\u0027t think it\u0027s an issue because once vrrp process dumps the pid file, it should be caught by the process manager. Before that, it will just fail to spawn keepalived, but since we retry, it should not be a great deal.","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"2aaa95092de530e7ca837d5aadb4a1509c5b360e","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"        # Exit the process, and see that when it comes back"},{"line_number":68,"context_line":"        # It\u0027s indeed a different process"},{"line_number":69,"context_line":"        utils.execute([\u0027kill\u0027, \u0027-15\u0027, pid], run_as_root\u003dTrue)"},{"line_number":70,"context_line":"        utils.wait_until_true("},{"line_number":71,"context_line":"            lambda: process.active and pid !\u003d process.pid,"},{"line_number":72,"context_line":"            timeout\u003d5,"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_2e9efc63","line":69,"in_reply_to":"7a740942_581c72c1","updated":"2015-12-09 14:26:36.000000000","message":"-9 plus the code here might make the functional test fail intermittently, I have tried it in PS1. \n\nIt looks like \"kill -9\" to keepalived can kill the vrrp process in test server(maybe that is why this case will not always fail before). \n\nBut there is a race condition. If the code here finds that keepalived is not active, and vrrp is active, then the code try to kill vrrp, however, the vrrp process just exist after the active check. This will make the killing vrrp fail, which will fail the test case.\n\nIn my PS1, it is very likely to happen the problem, because the respawn action is right after the \"kill -9\". Here is asynchronous. But I still see a chance to the race condition. Do you think it is worth a try here?","commit_id":"e92e381fa2f81c925ea2c31dfd827b3d1b2a8180"},{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"8e5fff6ecde37a480ad1dd4b943381ba3b5e18f8","unresolved":false,"context_lines":[{"line_number":75,"context_line":"            sleep\u003d0.01,"},{"line_number":76,"context_line":"            exception\u003dRuntimeError(_(\"Keepalived didn\u0027t respawn\")))"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"    def test_keepalived_respawns(self):"},{"line_number":79,"context_line":"        self._test_keepalived_respawns()"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"    def test_keepalived_respawn_with_unexpected_exit(self):"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a710552_5c21609e","line":78,"updated":"2015-12-16 10:44:23.000000000","message":"thanks for covering both.","commit_id":"08171f58e9073cc071151f265a25174fb0299aa6"}]}
