)]}'
{"doc/source/contributor/policies/gate-failure-triage.rst":[{"author":{"_account_id":17120,"name":"Manjeet Singh Bhatia","email":"manjeet.s.bhatia@intel.com","username":"manjeets"},"change_message_id":"a638fd17f01387492be5736d2ec6999309e1f722","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_35802ef5","line":77,"range":{"start_line":77,"start_character":23,"end_line":77,"end_character":32},"updated":"2018-04-03 15:53:29.000000000","message":"functional","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_16ea06d7","line":77,"range":{"start_line":77,"start_character":23,"end_line":77,"end_character":32},"in_reply_to":"bf659307_35802ef5","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0b327dd232bfa36aaa01c2e9cb376240828af70d","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_358b4ec3","line":79,"range":{"start_line":78,"start_character":47,"end_line":79,"end_character":76},"updated":"2018-04-03 15:55:20.000000000","message":"This to me is a clear indication that there is a bug in our logging. If I am an operating operating this software and cannot debug problems from the logs I would be very sad if the response is to be asked to run pdb.\n\nI think the first step and last step in each of these cases should be to evaluate the logging we have in place and address any deficiencies preventing the debugging from happening otherwise.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_16ed26b6","line":79,"range":{"start_line":78,"start_character":47,"end_line":79,"end_character":76},"in_reply_to":"bf659307_358b4ec3","updated":"2018-04-12 09:56:37.000000000","message":"yes, I totally agree with You. Please note that there is whole section just above this one about checking job logs to troubleshoot failing tests.\nI added some short info about that in this section also","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0b327dd232bfa36aaa01c2e9cb376240828af70d","unresolved":false,"context_lines":[{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_15708a07","line":99,"range":{"start_line":99,"start_character":8,"end_line":99,"end_character":85},"updated":"2018-04-03 15:55:20.000000000","message":"You should not open the entire host to any tcp connection. We run services on these hosts that would be accessible in this case that could potentially be abused for things like reflection attacks.\n\nPlease add the specific port that you need explicitly instead.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_d6f56e58","line":99,"range":{"start_line":99,"start_character":8,"end_line":99,"end_character":85},"in_reply_to":"bf659307_15708a07","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0b327dd232bfa36aaa01c2e9cb376240828af70d","unresolved":false,"context_lines":[{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_d567724c","line":106,"updated":"2018-04-03 15:55:20.000000000","message":"Note the overall job will be limited by the job timeout and that can\u0027t be changed from within the job.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_f6f9924d","line":106,"in_reply_to":"bf659307_d567724c","updated":"2018-04-12 09:56:37.000000000","message":"yes, I know. I added comment about that also. Thx","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_a042be0d","line":109,"range":{"start_line":109,"start_character":57,"end_line":109,"end_character":65},"updated":"2018-04-03 16:32:22.000000000","message":"checking the","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_56863ebb","line":109,"range":{"start_line":109,"start_character":57,"end_line":109,"end_character":65},"in_reply_to":"bf659307_a042be0d","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_1b167b4a","line":118,"range":{"start_line":118,"start_character":60,"end_line":118,"end_character":70},"updated":"2018-04-03 16:32:22.000000000","message":"starts","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_168c46d5","line":118,"range":{"start_line":118,"start_character":60,"end_line":118,"end_character":70},"in_reply_to":"bf659307_1b167b4a","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_c007b2c9","line":142,"range":{"start_line":142,"start_character":12,"end_line":142,"end_character":15},"updated":"2018-04-03 16:32:22.000000000","message":"note","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_40fba2bf","line":144,"range":{"start_line":144,"start_character":72,"end_line":144,"end_character":79},"updated":"2018-04-03 16:32:22.000000000","message":"method","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_d6680ef6","line":144,"range":{"start_line":144,"start_character":72,"end_line":144,"end_character":79},"in_reply_to":"bf659307_40fba2bf","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_00012aaf","line":149,"range":{"start_line":149,"start_character":18,"end_line":149,"end_character":25},"updated":"2018-04-03 16:32:22.000000000","message":"is","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_36770ad7","line":149,"range":{"start_line":149,"start_character":18,"end_line":149,"end_character":25},"in_reply_to":"bf659307_00012aaf","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_60cee65a","line":167,"range":{"start_line":167,"start_character":73,"end_line":167,"end_character":77},"updated":"2018-04-03 16:32:22.000000000","message":"not needed","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_b64c7a9e","line":167,"range":{"start_line":167,"start_character":73,"end_line":167,"end_character":77},"in_reply_to":"bf659307_60cee65a","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_60f7c68c","line":170,"range":{"start_line":170,"start_character":66,"end_line":170,"end_character":71},"updated":"2018-04-03 16:32:22.000000000","message":"Not needed","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_d651ee37","line":170,"range":{"start_line":170,"start_character":66,"end_line":170,"end_character":71},"in_reply_to":"bf659307_60f7c68c","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":4694,"name":"Miguel Lavalle","email":"miguel@mlavalle.com","username":"minsel"},"change_message_id":"1afd351b57ca3b6b75d220e940601a21fab3dab1","unresolved":false,"context_lines":[{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_603ca62e","line":171,"range":{"start_line":171,"start_character":15,"end_line":171,"end_character":74},"updated":"2018-04-03 16:32:22.000000000","message":"I would rephrase this as: so they can unlock and destroy this hold node","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_b631da0d","line":171,"range":{"start_line":171,"start_character":15,"end_line":171,"end_character":74},"in_reply_to":"bf659307_603ca62e","updated":"2018-04-12 09:56:37.000000000","message":"Done","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":17120,"name":"Manjeet Singh Bhatia","email":"manjeet.s.bhatia@intel.com","username":"manjeets"},"change_message_id":"a638fd17f01387492be5736d2ec6999309e1f722","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"},{"line_number":83,"context_line":"might be necessary to access directly to node and debug ``live`` what"},{"line_number":84,"context_line":"happens there during  test."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"This can be done in two ways:"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"1. Using `remote_pdb \u003chttps://pypi.python.org/pypi/remote-pdb\u003e`_ python"},{"line_number":89,"context_line":"   module and access, using ``telnet`` directly to python debugger in failed"},{"line_number":90,"context_line":"   test."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   To achieve that, You need to send to gerrit ``Do not merge`` patch with"},{"line_number":93,"context_line":"   changes as below:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"   * add iptables rule to accept incomming telnet connection to remote_pdb. This"},{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."},{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"    It can be added in same section as described above OS_TEST_TIMEOUT change"},{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * finally, You need to import and call remote_pdb module in place of"},{"line_number":124,"context_line":"     Your code or test where You want to start debugger::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"        diff --git a/neutron/tests/fullstack/test_connectivity.py b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":127,"context_line":"        index c8650b0..260207b 100644"},{"line_number":128,"context_line":"        --- a/neutron/tests/fullstack/test_connectivity.py"},{"line_number":129,"context_line":"        +++ b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":130,"context_line":"        @@ -189,6 +189,11 @@ class"},{"line_number":131,"context_line":"        TestLinuxBridgeConnectivitySameNetwork(BaseConnectivitySameNetworkTest):"},{"line_number":132,"context_line":"                ]"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"             def test_connectivity(self):"},{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":157,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":160,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":161,"context_line":"   important for futher debugging."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"},{"line_number":175,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``,"},{"line_number":176,"context_line":"   * using ``SSH`` to connect as a root to test node."},{"line_number":177,"context_line":"   You can then ask infra-team to add You key to specific node on which You have"},{"line_number":178,"context_line":"   already started remote_pdb session also."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"Root Causing a Gate Failure"},{"line_number":181,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_95849ad4","line":178,"range":{"start_line":77,"start_character":1,"end_line":178,"end_character":43},"updated":"2018-04-03 15:53:29.000000000","message":"sounds very time consuming process, or would you mind adding something to debug locally ? to debug this with devstack by just having debug env in tox.ini\n\n[testenv:debug]\n# Uses default base python\nsetenv \u003d {[testenv]setenv}\ncommands \u003d oslo_debug_helper -t neutron/tests {posargs}\n\n\nset pdb traces in code and run tox -e debug regex_to_test.\n\nif you want to run debug a tempest test, then just set a break point in the tempest test, make sure devstack is up and running\n\nrun ostestr --pdb regex_to_test from tempest dir.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":17120,"name":"Manjeet Singh Bhatia","email":"manjeet.s.bhatia@intel.com","username":"manjeets"},"change_message_id":"0b6e424244e0271d75866ea702c00d98a9198df9","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"},{"line_number":83,"context_line":"might be necessary to access directly to node and debug ``live`` what"},{"line_number":84,"context_line":"happens there during  test."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"This can be done in two ways:"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"1. Using `remote_pdb \u003chttps://pypi.python.org/pypi/remote-pdb\u003e`_ python"},{"line_number":89,"context_line":"   module and access, using ``telnet`` directly to python debugger in failed"},{"line_number":90,"context_line":"   test."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   To achieve that, You need to send to gerrit ``Do not merge`` patch with"},{"line_number":93,"context_line":"   changes as below:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"   * add iptables rule to accept incomming telnet connection to remote_pdb. This"},{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."},{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"    It can be added in same section as described above OS_TEST_TIMEOUT change"},{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * finally, You need to import and call remote_pdb module in place of"},{"line_number":124,"context_line":"     Your code or test where You want to start debugger::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"        diff --git a/neutron/tests/fullstack/test_connectivity.py b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":127,"context_line":"        index c8650b0..260207b 100644"},{"line_number":128,"context_line":"        --- a/neutron/tests/fullstack/test_connectivity.py"},{"line_number":129,"context_line":"        +++ b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":130,"context_line":"        @@ -189,6 +189,11 @@ class"},{"line_number":131,"context_line":"        TestLinuxBridgeConnectivitySameNetwork(BaseConnectivitySameNetworkTest):"},{"line_number":132,"context_line":"                ]"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"             def test_connectivity(self):"},{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":157,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":160,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":161,"context_line":"   important for futher debugging."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"},{"line_number":175,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``,"},{"line_number":176,"context_line":"   * using ``SSH`` to connect as a root to test node."},{"line_number":177,"context_line":"   You can then ask infra-team to add You key to specific node on which You have"},{"line_number":178,"context_line":"   already started remote_pdb session also."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"Root Causing a Gate Failure"},{"line_number":181,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_dbe243d5","line":178,"range":{"start_line":77,"start_character":1,"end_line":178,"end_character":43},"in_reply_to":"bf659307_3527cecc","updated":"2018-04-03 16:45:09.000000000","message":"that can happen for range of issues, like dependency mismatch, infra issues and so on, pdb make more sense for debugging neutron code here. not sure how possible this is security wise to do over gate and also depends on availability of infra folks. In case dev\u0027s want to quick try then information on how to debug locally could be handy as well.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"c799a94da17101250c7f4af4c01f9b075b7bc23f","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"},{"line_number":83,"context_line":"might be necessary to access directly to node and debug ``live`` what"},{"line_number":84,"context_line":"happens there during  test."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"This can be done in two ways:"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"1. Using `remote_pdb \u003chttps://pypi.python.org/pypi/remote-pdb\u003e`_ python"},{"line_number":89,"context_line":"   module and access, using ``telnet`` directly to python debugger in failed"},{"line_number":90,"context_line":"   test."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   To achieve that, You need to send to gerrit ``Do not merge`` patch with"},{"line_number":93,"context_line":"   changes as below:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"   * add iptables rule to accept incomming telnet connection to remote_pdb. This"},{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."},{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"    It can be added in same section as described above OS_TEST_TIMEOUT change"},{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * finally, You need to import and call remote_pdb module in place of"},{"line_number":124,"context_line":"     Your code or test where You want to start debugger::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"        diff --git a/neutron/tests/fullstack/test_connectivity.py b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":127,"context_line":"        index c8650b0..260207b 100644"},{"line_number":128,"context_line":"        --- a/neutron/tests/fullstack/test_connectivity.py"},{"line_number":129,"context_line":"        +++ b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":130,"context_line":"        @@ -189,6 +189,11 @@ class"},{"line_number":131,"context_line":"        TestLinuxBridgeConnectivitySameNetwork(BaseConnectivitySameNetworkTest):"},{"line_number":132,"context_line":"                ]"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"             def test_connectivity(self):"},{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":157,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":160,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":161,"context_line":"   important for futher debugging."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"},{"line_number":175,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``,"},{"line_number":176,"context_line":"   * using ``SSH`` to connect as a root to test node."},{"line_number":177,"context_line":"   You can then ask infra-team to add You key to specific node on which You have"},{"line_number":178,"context_line":"   already started remote_pdb session also."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"Root Causing a Gate Failure"},{"line_number":181,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_3527cecc","line":178,"range":{"start_line":77,"start_character":1,"end_line":178,"end_character":43},"in_reply_to":"bf659307_95849ad4","updated":"2018-04-03 15:58:04.000000000","message":"sure, it is time consuming process but please keep in mind that this isn\u0027t about debugging the case when You can reproduce issue locally - that is quite easy to fix.\nThis is only for the cases when test is failing only sometimes on the test nodes but it\u0027s not failing locally (we had many such cases recently). Then as last chance You can try to do something like is described here to debug it.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"067b02e4e328148cfaecb996078c980c9b7d4a79","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Advanced troubleshooting gate jobs"},{"line_number":76,"context_line":"----------------------------------"},{"line_number":77,"context_line":"Sometimes when tempest/functiona/fullstack job is failing often, it might be"},{"line_number":78,"context_line":"hard to reproduce it in local environments and it might be also very hard to"},{"line_number":79,"context_line":"understand the reason of such failures only from reading logs of failed job."},{"line_number":80,"context_line":"In such case there are some additional possiblilities to debug test directly"},{"line_number":81,"context_line":"on test node."},{"line_number":82,"context_line":"To understand exactly what happens during such failing test on test node it"},{"line_number":83,"context_line":"might be necessary to access directly to node and debug ``live`` what"},{"line_number":84,"context_line":"happens there during  test."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"This can be done in two ways:"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"1. Using `remote_pdb \u003chttps://pypi.python.org/pypi/remote-pdb\u003e`_ python"},{"line_number":89,"context_line":"   module and access, using ``telnet`` directly to python debugger in failed"},{"line_number":90,"context_line":"   test."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   To achieve that, You need to send to gerrit ``Do not merge`` patch with"},{"line_number":93,"context_line":"   changes as below:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"   * add iptables rule to accept incomming telnet connection to remote_pdb. This"},{"line_number":96,"context_line":"     can be done e.g. in ``neutron/tests/contrib/post_test_hook.sh`` file, in"},{"line_number":97,"context_line":"     proper section according to test which You want to debug::"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"        sudo iptables -I openstack-INPUT -p tcp -m state --state NEW -m tcp -j ACCEPT"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   * increase ``OS_TEST_TIMEOUT`` to make test stay longer with remote_pdb"},{"line_number":102,"context_line":"     active and make debugging easier."},{"line_number":103,"context_line":"     This change can be also done in ``neutron/tests/contrib/post_test_hook.sh``"},{"line_number":104,"context_line":"     file, in same section as above adding of iptables rule::"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"        export OS_TEST_TIMEOUT\u003d999999"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"   * to make easier finding IP address of test node, You can also add to"},{"line_number":109,"context_line":"     ``/neutron/tests/contrib/post_test_hook.sh`` script check of IPs configured"},{"line_number":110,"context_line":"     on test node, for example::"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"        hostname -I"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"    It can be added in same section as described above OS_TEST_TIMEOUT change"},{"line_number":115,"context_line":"    and iptables change."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"   * add package ``remote_pdb`` to ``test-requirements.txt`` file. Then it will"},{"line_number":118,"context_line":"     be automatically installed in test\u0027s venv before tests will start::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         cat test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * finally, You need to import and call remote_pdb module in place of"},{"line_number":124,"context_line":"     Your code or test where You want to start debugger::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"        diff --git a/neutron/tests/fullstack/test_connectivity.py b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":127,"context_line":"        index c8650b0..260207b 100644"},{"line_number":128,"context_line":"        --- a/neutron/tests/fullstack/test_connectivity.py"},{"line_number":129,"context_line":"        +++ b/neutron/tests/fullstack/test_connectivity.py"},{"line_number":130,"context_line":"        @@ -189,6 +189,11 @@ class"},{"line_number":131,"context_line":"        TestLinuxBridgeConnectivitySameNetwork(BaseConnectivitySameNetworkTest):"},{"line_number":132,"context_line":"                ]"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"             def test_connectivity(self):"},{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"     Please not that discovery of public IP address is necessary because by"},{"line_number":143,"context_line":"     default remote_pdb will bind only to ``127.0.0.1`` IP address."},{"line_number":144,"context_line":"     You can do it in any way. Above is just example of one of possible methods."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"   Now, when You have all those changes done, You can commit it and go to `Zuul"},{"line_number":147,"context_line":"   status page \u003chttps://zuul.openstack.org\u003e`_ to find status of tests for Your"},{"line_number":148,"context_line":"   ``DNM`` patch, then open console log from Your job and wait there until"},{"line_number":149,"context_line":"   ``remote_pdb`` will be started."},{"line_number":150,"context_line":"   You need to find in this console log IP address of test node also. This is"},{"line_number":151,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":152,"context_line":"   something like::"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":157,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":160,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":161,"context_line":"   important for futher debugging."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":164,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"},{"line_number":165,"context_line":"   so on, You can ask members of infra-team to hold off troubleshooted job."},{"line_number":166,"context_line":"   You can ask for that on ``openstack-infra`` on IRC."},{"line_number":167,"context_line":"   In such case infra-team will add Your SSH key to test node and if job will"},{"line_number":168,"context_line":"   fails, node will not be destroyed. You will be able to SSH to it and debug"},{"line_number":169,"context_line":"   what You need."},{"line_number":170,"context_line":"   Please remember to tell on ``openstack-infra`` channel when You will finish"},{"line_number":171,"context_line":"   Your debug, that they will be able to unlock and destroy this hold node."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"   Above two solutions can be used together. Than You might be able to connect"},{"line_number":174,"context_line":"   to test node with both methods:"},{"line_number":175,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``,"},{"line_number":176,"context_line":"   * using ``SSH`` to connect as a root to test node."},{"line_number":177,"context_line":"   You can then ask infra-team to add You key to specific node on which You have"},{"line_number":178,"context_line":"   already started remote_pdb session also."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"Root Causing a Gate Failure"},{"line_number":181,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_9615b64b","line":178,"range":{"start_line":77,"start_character":1,"end_line":178,"end_character":43},"in_reply_to":"bf659307_dbe243d5","updated":"2018-04-12 09:56:37.000000000","message":"I agree that is should be reviewed by infra team if they are fine with publishing something like that so I asked for such review already :)\nAbout local debugging - please note that this is section about troubleshooting gate jobs not debugging in overall.\nMaybe we can consider to add some guide how to debug neutron code/tests locally but IMO it\u0027s not for this patch.","commit_id":"6b248d4aec2634c159ee9e50c27ec9a88d1027a6"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"673de3d2f1b7b5ff97382f0b0177f71031fda145","unresolved":false,"context_lines":[{"line_number":156,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":157,"context_line":"   something like::"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":162,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f6a8fd7_95804e48","line":159,"range":{"start_line":159,"start_character":47,"end_line":159,"end_character":52},"updated":"2018-04-23 11:30:57.000000000","message":"Small thing: this should be 44444 as in l143, am I wrong?","commit_id":"326819b6a8a86b7dc8f01392e646d1ace260ba4d"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"86e8c1ad2bf59b46fee469fcd7b09959cec5c5c6","unresolved":false,"context_lines":[{"line_number":156,"context_line":"   necessary to connect via ``telnet`` and start debugging. It should be"},{"line_number":157,"context_line":"   something like::"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        RemotePdb session open at 172.99.68.50:34536, waiting for connection ..."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"   Example of such DNM patch described above can be seen on"},{"line_number":162,"context_line":"   `\u003chttps://review.openstack.org/#/c/558259/\u003e`_."}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f6a8fd7_315832f8","line":159,"range":{"start_line":159,"start_character":47,"end_line":159,"end_character":52},"in_reply_to":"9f6a8fd7_95804e48","updated":"2018-04-25 20:26:12.000000000","message":"right","commit_id":"326819b6a8a86b7dc8f01392e646d1ace260ba4d"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"673de3d2f1b7b5ff97382f0b0177f71031fda145","unresolved":false,"context_lines":[{"line_number":163,"context_line":""},{"line_number":164,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":165,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":166,"context_line":"   important for futher debugging."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":169,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f6a8fd7_158c3e7d","line":166,"range":{"start_line":166,"start_character":17,"end_line":166,"end_character":23},"updated":"2018-04-23 11:30:57.000000000","message":"typo: further","commit_id":"326819b6a8a86b7dc8f01392e646d1ace260ba4d"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"86e8c1ad2bf59b46fee469fcd7b09959cec5c5c6","unresolved":false,"context_lines":[{"line_number":163,"context_line":""},{"line_number":164,"context_line":"   Please note, that after adding some new packages to requirements file,"},{"line_number":165,"context_line":"   ``requirements-check`` job for Your test patch will fail but it\u0027s not"},{"line_number":166,"context_line":"   important for futher debugging."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"2. If root access to test node is necessary, for example to check if VMs are"},{"line_number":169,"context_line":"   really spawned, router/dhcp namespaces are really configured properly and"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f6a8fd7_310d92e7","line":166,"range":{"start_line":166,"start_character":17,"end_line":166,"end_character":23},"in_reply_to":"9f6a8fd7_158c3e7d","updated":"2018-04-25 20:26:12.000000000","message":"Done","commit_id":"326819b6a8a86b7dc8f01392e646d1ace260ba4d"},{"author":{"_account_id":17120,"name":"Manjeet Singh Bhatia","email":"manjeet.s.bhatia@intel.com","username":"manjeets"},"change_message_id":"54c09bbe648167e0b2dbff82d152d467af9c0793","unresolved":false,"context_lines":[{"line_number":76,"context_line":"-------------------------------------"},{"line_number":77,"context_line":"As a first step of troubleshooting a failing gate job, you should always check"},{"line_number":78,"context_line":"the logs of the job as described above."},{"line_number":79,"context_line":"Unfortunately, sometimes when a tempest/functional/fullstack jobs is"},{"line_number":80,"context_line":"failing, it might be hard to reproduce it in a local environment, and might"},{"line_number":81,"context_line":"also be very hard to understand the reason of such a failure from only reading"},{"line_number":82,"context_line":"the logs of the failed job.  In such cases there are some additional ways"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_161856d1","line":79,"range":{"start_line":79,"start_character":61,"end_line":79,"end_character":65},"updated":"2018-04-27 15:59:09.000000000","message":"job","commit_id":"87c00207db8c9d437db011a2560d8d10fd9d0374"},{"author":{"_account_id":841,"name":"Akihiro Motoki","email":"amotoki@gmail.com","username":"amotoki"},"change_message_id":"d18278d2a5fbeeb98ddb089a8b7017161a3b6127","unresolved":false,"context_lines":[{"line_number":108,"context_line":"     and that cannot be changed from within the job."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"   * To make it easier to find the IP address of the test node, you should"},{"line_number":111,"context_line":"     add a line to the ``/neutron/tests/contrib/post_test_hook.sh`` file to"},{"line_number":112,"context_line":"     print the IPs configured on the test node.  For example::"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        hostname -I"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_7f8e0828","line":111,"range":{"start_line":111,"start_character":25,"end_line":111,"end_character":26},"updated":"2018-05-01 23:33:14.000000000","message":"This slash looks unnecessary.","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"be4f739590b4a9f99c5faf85d5d22a64782a7861","unresolved":false,"context_lines":[{"line_number":108,"context_line":"     and that cannot be changed from within the job."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"   * To make it easier to find the IP address of the test node, you should"},{"line_number":111,"context_line":"     add a line to the ``/neutron/tests/contrib/post_test_hook.sh`` file to"},{"line_number":112,"context_line":"     print the IPs configured on the test node.  For example::"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        hostname -I"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_3cc5e73c","line":111,"range":{"start_line":111,"start_character":25,"end_line":111,"end_character":26},"in_reply_to":"9f6a8fd7_7f8e0828","updated":"2018-05-02 20:21:45.000000000","message":"Done","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":841,"name":"Akihiro Motoki","email":"amotoki@gmail.com","username":"amotoki"},"change_message_id":"d18278d2a5fbeeb98ddb089a8b7017161a3b6127","unresolved":false,"context_lines":[{"line_number":117,"context_line":"     That way it will be automatically installed in the venv of the test"},{"line_number":118,"context_line":"     before it is run::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         tail -1 test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * Finally, you need to import and call the remote_pdb module in the part"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_bfab4073","line":120,"range":{"start_line":120,"start_character":7,"end_line":120,"end_character":13},"updated":"2018-05-01 23:33:14.000000000","message":"It would be nice if you add a command prompt here. It is easier to understand.","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"be4f739590b4a9f99c5faf85d5d22a64782a7861","unresolved":false,"context_lines":[{"line_number":117,"context_line":"     That way it will be automatically installed in the venv of the test"},{"line_number":118,"context_line":"     before it is run::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"         tail -1 test-requirements.txt"},{"line_number":121,"context_line":"         remote_pdb"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   * Finally, you need to import and call the remote_pdb module in the part"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_fcdc4f19","line":120,"range":{"start_line":120,"start_character":7,"end_line":120,"end_character":13},"in_reply_to":"9f6a8fd7_bfab4073","updated":"2018-05-02 20:21:45.000000000","message":"Done","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":841,"name":"Akihiro Motoki","email":"amotoki@gmail.com","username":"amotoki"},"change_message_id":"d18278d2a5fbeeb98ddb089a8b7017161a3b6127","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip, port\u003d44444)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_bfc4a041","line":138,"range":{"start_line":138,"start_character":36,"end_line":138,"end_character":75},"updated":"2018-05-01 23:33:14.000000000","message":"Can\u0027t we listen 0.0.0.0 instead of my_ip? If possible, it would be simpler.","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"be4f739590b4a9f99c5faf85d5d22a64782a7861","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        +        from urllib import urlopen"},{"line_number":136,"context_line":"        +        from json import load"},{"line_number":137,"context_line":"        +        my_ip \u003d load(urlopen(\u0027https://api.ipify.org/?format\u003djson\u0027))[\u0027ip\u0027]"},{"line_number":138,"context_line":"        +        import remote_pdb; remote_pdb.set_trace(my_ip, port\u003d44444)"},{"line_number":139,"context_line":"        +"},{"line_number":140,"context_line":"        self._test_connectivity()"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_5cdf7bd5","line":138,"range":{"start_line":138,"start_character":36,"end_line":138,"end_character":75},"in_reply_to":"9f6a8fd7_bfc4a041","updated":"2018-05-02 20:21:45.000000000","message":"I tried it locally and it worked, so I\u0027ll make that change.","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":841,"name":"Akihiro Motoki","email":"amotoki@gmail.com","username":"amotoki"},"change_message_id":"d18278d2a5fbeeb98ddb089a8b7017161a3b6127","unresolved":false,"context_lines":[{"line_number":171,"context_line":"   and debug things further.  Please remember to tell the infra-team when you"},{"line_number":172,"context_line":"   finish debugging so they can unlock and destroy the node being held."},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"   The above two solutions can be used together. For example, you should be"},{"line_number":175,"context_line":"   able to connect to the test node with both methods:"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``;"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_9f08bc91","line":174,"range":{"start_line":174,"start_character":0,"end_line":174,"end_character":3},"updated":"2018-05-01 23:33:14.000000000","message":"This paragraph does not belong to the second item. We can drop the leading spaces.","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"be4f739590b4a9f99c5faf85d5d22a64782a7861","unresolved":false,"context_lines":[{"line_number":171,"context_line":"   and debug things further.  Please remember to tell the infra-team when you"},{"line_number":172,"context_line":"   finish debugging so they can unlock and destroy the node being held."},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"   The above two solutions can be used together. For example, you should be"},{"line_number":175,"context_line":"   able to connect to the test node with both methods:"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"   * using ``remote_pdb`` to connect via ``telnet``;"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9f6a8fd7_dce94bb7","line":174,"range":{"start_line":174,"start_character":0,"end_line":174,"end_character":3},"in_reply_to":"9f6a8fd7_9f08bc91","updated":"2018-05-02 20:21:45.000000000","message":"Done","commit_id":"d4596caa52d2bbd6942365bb009782043fe799ef"}]}
