)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"d81159cb0509168412b5e41b0908c8b38b8d87d3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"14ae7c5e_8d7d5cfd","updated":"2021-10-13 08:39:14.000000000","message":"Correct ovn test result link\n\n2, for ovn, it works well -  https://pastebin.ubuntu.com/p/NcMJW4r5sk/","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"53a3c0cec5d42b49beb5e5d9530839a6cb662cc0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"d0ae26a2_cd319467","updated":"2021-10-13 08:31:36.000000000","message":"FYI: the POST_FAILUREs are unrelated ongoing known upstream Zuul issues, being currently discussed on irc/oftc/#opendev","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"219e04223a557102d8faf8a7ba4c5549cad46ff6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"c846d09d_71b56877","updated":"2021-10-19 08:25:32.000000000","message":"If you replace /requirements.txt in this repo with the content of https://github.com/openstack-charmers/release-tools/blob/master/global/source-zaza/requirements.txt , this should fix the python 3.5 failures.","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"99dee69e1090e8660855d5f818a86064fc7514e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"ba80bb8a_14f75306","updated":"2021-10-14 01:17:10.000000000","message":"The following successful CI also prove that the current CI failure is not caused by my patch, so I triggered it again to see if there is good luck this time.\n\nbionic-train-ha-ovn https://openstack-ci-reports.ubuntu.com/artifacts/1fc/787700/22/check/bionic-train-ha-ovn/1fcadcc/ : SUCCESS in 3h 07m 00s","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c1a13866d63c793f354571f84572a06b3f3daced","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"20f4f44a_cf5275b6","updated":"2021-10-18 09:46:18.000000000","message":"The python3.5 failures are unrelated and affect all our OpenStack charms. cffi has dropped python3.5 upstream and we are trying to solve this from two different angles:\nhttps://review.opendev.org/c/openstack/openstack-zuul-jobs/+/814126\nhttps://review.opendev.org/q/topic:stable-2110-candidate","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"1f743235537a2c560e23e49578ed1a437512e8ea","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"65585dbc_04498333","updated":"2021-10-14 01:03:47.000000000","message":"charm-recheck","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"343d85f1807f1991d7bbcd06cff3f028de18bb95","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"c8da06fc_c1a40236","updated":"2021-10-13 13:29:55.000000000","message":"charm-recheck","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"013e08e41d8e2d1f0aa1dfded32435fc88fa6910","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"c9ca0c18_66e190cd","updated":"2021-10-18 06:48:44.000000000","message":"charm-recheck","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"379d079b67e8768451dd0806cca3e3709f7bd3e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"f1c82a79_4068a5cf","updated":"2021-10-18 09:36:42.000000000","message":"charm-recheck","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"3ebf2ea707dcdf41e946f1f03e2c86bc7b44bd27","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"ed0c59f6_fd632bd4","updated":"2021-10-13 08:27:29.000000000","message":"self-test results:\n\n1, for non-ovn, it works well -  https://pastebin.ubuntu.com/p/syJqbDh2SR/\n2, for ovn, it works well -  https://pastebin.ubuntu.com/p/syJqbDh2SR/\n\nBTW, why CI doesn\u0027t work for ovn before, that’s because all ports are DOWN for ovn - https://pastebin.ubuntu.com/p/s56yQWqJMj/\n\nMy CI code has logic to rule out DOWN port before - https://github.com/openstack-charmers/zaza-openstack-tests/pull/600/commits/b44aa8823c85730ccc1b4537dab56862f8a2605c#diff-dd771fdb22ae2ad7c1df95f200cdb0fe7c0432041db30d45e4c8317c7c272894R118\n\nWhy I added this logic before, that’s because for non-ovn env, a new DOWN port may be recreated after deleting one unit - https://pastebin.ubuntu.com/p/YSSDGHJhr8/","commit_id":"2321b08077c83769cfc2c215928c7c375ea9d700"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"60b510b26b78b5ef21aea33a71540333b9629486","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"18df8f39_4b64e8e9","updated":"2021-10-25 04:12:36.000000000","message":"All python3.5 related patches are merged, so let me try again.\n\nhttps://review.opendev.org/c/openstack/openstack-zuul-jobs/+/814126\nhttps://review.opendev.org/q/topic:stable-2110-candidate","commit_id":"3aec3aa0c4b9aa27efbcc22268acca3d9501822c"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"358fcdc0891cd468e52c4875085f2cf75549b1d4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"649f8649_9742181b","updated":"2021-10-21 03:11:33.000000000","message":"Hi Aurelien, thanks, I have modified requirements.txt as you said.","commit_id":"3aec3aa0c4b9aa27efbcc22268acca3d9501822c"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"1ea453b78e3c22033436d6b2f4b74a41cf32e76a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"32f0f2e3_a5a25754","updated":"2021-10-25 04:12:50.000000000","message":"charm-recheck","commit_id":"3aec3aa0c4b9aa27efbcc22268acca3d9501822c"},{"author":{"_account_id":10366,"name":"Hemanth N","email":"hemanth.nakkina@canonical.com","username":"Hemanth"},"change_message_id":"f955aa5b7bbc2ee0df950dc8ba5cbb9016660ddc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"cc9da0e0_ef3727f8","updated":"2021-10-26 04:32:52.000000000","message":"recheck","commit_id":"3aec3aa0c4b9aa27efbcc22268acca3d9501822c"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"203c5911cefb7f2e559479dc12da8ea3f7bb102a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"797f8c97_3b1f94a2","updated":"2021-10-26 06:00:16.000000000","message":"For python3.5 failures, move the following line from requirements.txt to test-requirements.txt to try again.\n\ncffi\u003d\u003d1.14.6; python_version \u003c \u00273.6\u0027  # cffi 1.15.0 drops support for py35.","commit_id":"9cec1b94bd96e67ef3d5f76be3b52813e205abd0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"2c3eb9a3ca2fc6f97724693c60b734031442c9e7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"4f2375fb_8ebaad35","updated":"2021-10-26 08:10:17.000000000","message":"This time the error \u0027c/_cffi_backend.c:15:17: fatal error: ffi.h: No such file or directory\u0027 happened during charm-build perirod, so let us put the following line in requirements.txt per hemanth\u0027s suggestion, thanks hemanth\n\ncffi\u003d\u003d1.14.6; python_version \u003c \u00273.6\u0027  # cffi 1.15.0 drops support for py35.","commit_id":"9cec1b94bd96e67ef3d5f76be3b52813e205abd0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"86b72f2c17c26808c85b76d5c63ea4cb20b7c551","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"05e28550_4fddf37b","updated":"2021-10-27 03:05:04.000000000","message":"CI build for charm-octavia failed with pyparsing error this time, so add the following line into test-requirements.txt, thanks hemanth\n\ncffi\u003d\u003d1.14.6; python_version \u003c \u00273.6\u0027  # cffi 1.15.0 drops support for py35.","commit_id":"af761a56bb20e7bb19dd218935af05e1afd88894"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"f5f77889c5f483b8f9766e54c286a45fc83a7bd0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"05499d0f_831f71af","updated":"2021-10-28 03:18:13.000000000","message":"recheck","commit_id":"af761a56bb20e7bb19dd218935af05e1afd88894"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"b892c2b8837b17cba2cea777ce0e367bfd7ddf1c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"43895094_da14e358","updated":"2021-10-28 03:18:00.000000000","message":"the error is below, so will try again\n\n\u0027ERROR failed to bootstrap model: cannot start bootstrap instance in availability zone \"nova\": cannot run instance:  with fault \"No valid host was found. There are not enough hosts available.\"","commit_id":"af761a56bb20e7bb19dd218935af05e1afd88894"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"7e95a0cb01e0f47b16ad108b2a38b717d424c2b1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"05acdf55_af0b1f21","updated":"2021-10-28 05:56:27.000000000","message":"BTW, pyparsing patch is from \n\nhttps://review.opendev.org/c/openstack/charm-rabbitmq-server/+/815258/2/test-requirements.txt\n\nand \n\nhttps://review.opendev.org/c/openstack/charm-glance/+/814882/6/test-requirements.txt","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c8226549eef65c61a09caf97d61d39ab2e637c1f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"81912109_4a2c46c7","updated":"2021-11-04 09:46:10.000000000","message":"Thanks a lot for your hard work! I have re-reviewed the latest version of the entire patch and have discovered a couple of nits. Apart from that this looks good to me.","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"ffbb40cfc275483dcaf7835e1e7bb183f131126c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"fe3c21e3_77e31cfc","updated":"2021-11-04 01:34:55.000000000","message":"This time bionic-train-ha-ovn bionic-ussuri-ha-ovn, focal-ussuri-ha-ovn and focal-victoria-ha-ovn pass CI, \nbut focal-victoria-no-amphora failed due to timeout,\nand groovy-victoria-ha-ovn failed due to https://pastebin.ubuntu.com/p/KwxcdzKY54/ , \nso I retriggered CI again.","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"78165ca189961b4feba7273608c6bba12aad7ff8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"24fb1452_32829224","updated":"2021-11-03 07:08:19.000000000","message":"charm-recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a1a7447f6df1d548eaf69af8f73547776d549c2c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"42476429_5f4ed12f","updated":"2021-11-01 07:06:27.000000000","message":"charm-recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"ad3f454d12671c79a7e9e732eae0857bd6632d3e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"d418045c_f588f922","updated":"2021-11-04 01:29:12.000000000","message":"charm-recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"0d9a1ed1983fb8c00a853ff8562c73e07095d931","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"f448742e_e4551eb9","updated":"2021-11-02 07:49:57.000000000","message":"charm-recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"3541046b5040129f29d7e42292619c047fec9284","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"0c33e813_200b99a6","updated":"2021-11-01 11:11:30.000000000","message":"it\u0027s quota issue now\n\n2021-11-01 07:31:36.948018 | bionic-medium | caused by: request (http://10.245.161.158:8774/v2.1/servers) returned unexpected status: 403; error info: {\"forbidden\": {\"code\": 403, \"message\": \"Quota exceeded for cores: Requested 4, but already used 1000 of 1000 cores\"}}","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"cbb9f57c19772ab00bfd44903176868848c98d48","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"aedf94a6_3e4c3919","updated":"2021-11-03 07:08:00.000000000","message":"it\u0027s still quota error, so retrigger again\n\nhttps://openstack-ci-reports.ubuntu.com/artifacts/097/787700/27/check/bionic-train-ha-ovn/097f41b/log/juju-status.zaza-b5d9f83adf60.txt","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"d803a056e3800a2d6b0a68192325d186d3300bc4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"263fd35b_67692ff4","updated":"2021-10-28 13:26:29.000000000","message":"just now rebased zaza-openstack-tests patch [1] to include pyparsing patch [2], so trigger this CI again\n\n[1] https://github.com/openstack-charmers/zaza-openstack-tests/pull/600/\n[2] https://github.com/openstack-charmers/zaza-openstack-tests/pull/659/commits/ae50ed3199dac6f27d5d3d17c30cb00805cf0b27","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"28638d32c3cb6cf3d25f287c75802e447a116b60","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"b17ae6c2_66c7dbc4","updated":"2021-10-29 06:00:53.000000000","message":"pyparsing issue has been resolved now after rebasing zaza-openstack-tests patch, but CI keep going wrong with the following error:\n\n2021-10-28 14:19:11.703248 | bionic-medium | zaza.model.MachineError: Machines 10 in error state","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"9cf9d5c1a289062e991576c8145a4411946da280","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"eac616b7_b5b52199","updated":"2021-10-28 05:55:08.000000000","message":"pyparsing issue still exists, but the following log shows \u0027pyparsing\u003c3.0.0\u0027 in test-requirements.txt doesn\u0027t work, so copy this line into requirements.txt as well\n\n2021-10-28 03:33:24.307054 | bionic-medium | Collecting pyparsing\u003e\u003d2.0.2 (from packaging\u003e\u003d14-\u003etox)\n2021-10-28 03:33:24.726388 | bionic-medium |   Downloading https://files.pythonhosted.org/packages/1b/19/8d24092f75e3bc7b4b2a78c9c63664bdeb00298acd15fad5d614da2a25b1/pyparsing-3.0.3-py3-none-any.whl (96kB)","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"5963cc358bbc207243dbfa47771d3ef2a9d9eb96","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"37da6ed1_3fd57a2f","updated":"2021-10-29 06:01:02.000000000","message":"recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"8e294c2f9158ad2cb4808f194442a7e138d96606","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"984db817_5d5761c4","updated":"2021-10-28 13:26:40.000000000","message":"recheck","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a09bd9eafac67d25b52e2c1071b264203ebdb63c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"42801235_0f4d7c2d","updated":"2021-11-08 04:10:28.000000000","message":"Aurelien, I\u0027ve deleived your comments, thanks for your review.","commit_id":"80a91a1c6858c044b88003ee18995b842565906b"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"f4a87ca310399f366ed9737105417b1d2f97ff75","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"a4500b39_b973df1b","updated":"2021-11-08 08:50:57.000000000","message":"LGTM, thanks! However the new test test_update_controller_ip_port_list failed on:\n\nbionic-train-ha-ovn https://openstack-ci-reports.ubuntu.com/artifacts/8c6/787700/28/check/bionic-train-ha-ovn/8c685d7/ : FAILURE in 2h 27m 55s\n\n    FAIL: test_update_controller_ip_port_list (zaza.openstack.charm_tests.octavia.tests.CharmOperationTest)\n    Test update_controller_ip_port_list.\n    ----------------------------------------------------------------------\n    Traceback (most recent call last):\n      File \"/home/ubuntu/src/review.opendev.org/openstack/charm-octavia/build/builds/octavia/.tox/func-target/lib/python3.6/site-packages/zaza/openstack/charm_tests/octavia/tests.py\", line 154, in test_update_controller_ip_port_list\n        self.assertTrue(len(ips) \u003d\u003d num)\n    AssertionError: False is not true\n","commit_id":"80a91a1c6858c044b88003ee18995b842565906b"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"50d36df5ca476f7563606988ecce693a41c222e4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"13519768_e6f6faac","updated":"2021-11-18 07:48:29.000000000","message":"@Aurelien, I investigated timeout issue on bionic-ussuri-ha-ovn2\n\nIt\u0027s really running test_update_controller_ip_port_list, the unit status is \u0027not idle\u0027 after adding one octavia unit.\nI think it also shows that CI timeout issue has nothing to do with my code, because the flag unit.is.departing can only be set after deleting one octavia unit, not adding one octavia unit. The following data also proved this point.\n\nzhhuabj@rotom:~/bugs/320434/4d46f4e0-bc78-4702-92bf-c7e9f723e51c$ grep -r \u0027unit.is.departing\u0027 debug_log.txt \n\u003cemtpy\u003e\nzhhuabj@rotom:~/bugs/320434/4d46f4e0-bc78-4702-92bf-c7e9f723e51c$ grep -r \u0027entering setup_hm_port\u0027 octavia_*/var/log/juju |grep -v None\n\u003cemtpy\u003e\n\nThe following log shows it\u0027s caused by VIP,\n\noctavia/3                        blocked   executing  17       172.16.0.138    9876/tcp           \u0027amqp\u0027 incomplete, \u0027sdn-subordinate\u0027 missing, Virtual network for access to Amphorae is down\n  hacluster-octavia/3            waiting   executing           172.16.0.138                       Resource: res_octavia_7158c25_vip not yet configured\n\nSo I continued to check hacluster-octavia/3\u0027s log, \n\n2021-11-17 11:03:52 WARNING hanode-relation-changed Job for corosync.service failed because a timeout was exceeded.\n\nHere are logs from syslog,\n\nNov 17 10:36:48 juju-f5b51f-zaza-4a779b337a66-17 corosync[26093]:  [TOTEM ] A new membership (172.16.0.138:96) was formed. Members\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:    error: Resource start-up disabled since no STONITH resources have been defined\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:    error: Either configure some or disable STONITH with the stonith-enabled option\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:    error: NOTE: Clusters with shared data need STONITH to ensure data integrity\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:   notice: Delaying fencing operations until there are resources to manage\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:   notice: Scheduling Node juju-f5b51f-zaza-4a779b337a66-17 for shutdown\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:   notice:  * Shutdown juju-f5b51f-zaza-4a779b337a66-17\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:   notice: Calculated transition 1, saving inputs in /var/lib/pacemaker/pengine/pe-input-3.bz2\nNov 17 10:36:49 juju-f5b51f-zaza-4a779b337a66-17 pengine[26152]:   notice: Configuration ERRORs found during PE processing.  Please run \"crm_verify -L\" to identify issues.\n\nbut there is no more corosync\u0027s configuration data in sosreport. Do you have any idea for it ?","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"0cff0b606767507e7701888a3f6433aaf5d3f1b2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"4a03f3ee_5c7b0437","updated":"2021-11-12 01:25:29.000000000","message":"All following CI failures are not caused by this patch, it seems to be unstable, so will retrigger again\n\n1, for focal-ussuri-ha-ovn\n\n2021-11-11 22:30:59.844905 | bionic-medium | juju.errors.JujuError: command failed: [\u0027scp\u0027, \u0027-i\u0027, \u0027/home/ubuntu/.local/share/juju/ssh/juju_id_rsa\u0027, \u0027-o\u0027, \u0027StrictHostKeyChecking\u003dno\u0027, \u0027-q\u0027, \u0027-B\u0027, \u0027/tmp/tmpi8f8swrk\u0027, \u0027ubuntu@172.16.0.53:/home/ubuntu/60-dataport.yaml\u0027]\n\n2, for focal-victoria-ha-ovn\n\novn-central/2                    error     idle   15       172.16.0.43     6641/tcp,6642/tcp  hook failed: \"update-status\"\n2021-11-11 22:47:25.842636 | bionic-medium | zaza.model.UnitError: Units ovn-central/2 in error state\n\n3, for focal-victoria-no-amphora\n\nkeystone/0*               error        idle        1        172.16.0.180              hook failed: \"install\"\n\n4, for groovy-victoria-ha-ovn\n\noctavia/1                        error     idle   11       172.16.0.180    9876/tcp           hook failed: \"identity-service-relation-changed\"","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"f95f1b1f5c0abe136f208a5dec129c304ec2c14a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"e1a8b464_4c47e4ee","updated":"2021-11-23 06:52:34.000000000","message":"Hi Aurelien,\n\nCI timeout issue is not caused by my test, so just like what happended above disabling my test will not prevent CI timeout issue as well.\n\nCI timeout issue is caused by hacluster issue (https://bugs.launchpad.net/charm-hacluster/+bug/1951392), so do we have any way to make CI not trigger the HA mode?","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"b946b68d468acf204aa6bebc5743b69387e72f35","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"3f2cb087_5a5cdbb3","updated":"2021-11-09 01:31:27.000000000","message":"Hi Aurelien, \n\nI have found the reason for the CI error \u0027AssertionError: False is not true\u0027, see https://pastebin.ubuntu.com/p/cxthjF9w8J/\n\nso L156 - L167 in octavia_handlers.py (https://review.opendev.org/c/openstack/charm-octavia/+/787700/28/src/reactive/octavia_handlers.py#167 ) can not completely prevent the previous DOWN port issue (the port got deleted by leader unit and then setup_hm_port might be called from departing unit to recreate a newly DOWN port)\n\nNow I am trying this solution - https://pastebin.ubuntu.com/p/RvfHwvHWRC/","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"b45300d8f50400d1322149ffee08b8e47535f8c2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"5a724250_5a3fc66d","updated":"2021-11-19 10:22:29.000000000","message":"Hi Aurelien, \n\nToday, I set up test env in stsstack and retest our code again, it works, see https://pastebin.ubuntu.com/p/3f7Ygmg9Tx/\n\nThen I ran it again, the same timeout issue was reproduced, see - https://pastebin.ubuntu.com/p/z8SwYR5cHy/\n\nbut it has nothing to do with our code, it\u0027s the same as the bug we reported before - https://bugs.launchpad.net/charm-hacluster/+bug/1951392 ","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"97adbb712fdb724b6d5de969fdb0ae87f30132f1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"7ba12461_8489ac05","updated":"2021-11-17 08:57:53.000000000","message":"Hi Aurelien, I don\u0027t think timeout issue is caused by my code, the reasons are as follows.\n\n1, The only difference between patchset28 and patchset29 is in the flag unit.is.departing ( https://pastebin.ubuntu.com/p/RvfHwvHWRC/ ), this flag is only added for departing unit to prevent creating newly DOWN hm port after leader unit has deleted hm port (the log shows this flag is only related to departing unit as well, flag should be unit isolated - https://pastebin.ubuntu.com/p/73GZXRn6Hm/), so I don\u0027t think adding this flag can cause timeout issue.\n\nWhy do I add it? pls refer to the previous comment. I pasted it below as well.\n\nI have found the reason for the CI error \u0027AssertionError: False is not true\u0027, see https://pastebin.ubuntu.com/p/cxthjF9w8J/\nso L156 - L167 in octavia_handlers.py (https://review.opendev.org/c/openstack/charm-octavia/+/787700/28/src/reactive/octavia_handlers.py#167 ) can not completely prevent the previous DOWN port issue (the port got deleted by leader unit and then setup_hm_port might be called from departing unit to recreate a newly DOWN port)\nNow I am trying this solution - https://pastebin.ubuntu.com/p/RvfHwvHWRC/\n\n2, In fact, we also have successful examples of running CI with my code. eg: focal-ussuri-ha-ovn, focal-wallaby-ha-ovn\n\n3, Yeah, for groovy-victoria-ha-ovn, it\u0027s really running my test_update_controller_ip_port_list, all units can\u0027t go back to idle/active/ready again after removing one unit.\n\n4, but for other tests, eg: bionic-train-ha-ovn, the same timeout issues was happening before entering my code.\n\n5, Other reviews (https://review.opendev.org/q/project:openstack%252Fcharm-octavia+branch:master ) also have similar problems, eg: https://review.opendev.org/c/openstack/charm-octavia/+/810567\n\nSo for the above reasons, I think timeout failures has nothing to do with my code. What\u0027s your opinion? ","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"fec4b04dcb8486f23822145ea91ea2ad98c5c28f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"eddeaea6_799cd39e","updated":"2021-11-18 08:49:37.000000000","message":"Hi Aurelien, pls see my two latest comments, and also this bug - https://bugs.launchpad.net/charm-hacluster/+bug/1951392 , thanks","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"d16187bd0edf8d08b8f5e81bd52ae7191c783b0d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"6338b3a6_e39cfc6c","updated":"2021-11-22 17:05:17.000000000","message":"Thanks a lot Zhang. This patch and the associated func-test-pr look good to me from a reviewer perspective. It seems indeed there no easy way for now to get this new test stable and this is not your code\u0027s fault. For example https://openstack-ci-reports.ubuntu.com/artifacts/3c7/787700/29/check/focal-ussuri-ha-ovn/3c70bc7/job-output.txt failed because of https://bugs.launchpad.net/charm-octavia/+bug/1951858\n\nI think we should disable your new test in the func-test-pr, while pointing at lp:1951858, and move on. Thank you!\n\nA disabled test looks like:\n\n    def test_mytest(self):\n        raise unittest.SkipTest(\"Skipping because of lp:1951858\")\n        ...\n","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"db3ac59bb64eafeb8ba26d5655513c5c88e3ef84","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"6da88f96_d2bce21d","updated":"2021-11-23 13:16:46.000000000","message":"The CI failures are infra issues. IMHO there has been enough successful runs and I\u0027m confident enough to land this change. Thanks a lot!","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"0b6cf150e09b30725f80bb59c3486064a3f01016","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"2d491fdd_85c657f7","updated":"2021-11-16 08:52:41.000000000","message":"The last run was timing out in test_update_controller_ip_port_list because after removing one unit, it is waiting for all units to become idle/active/ready again. However the hacluster-octavia units end up forever in \"waiting\" state, see https://openstack-ci-reports.ubuntu.com/artifacts/015/787700/29/check/groovy-victoria-ha-ovn/0156729/log/juju-status.zaza-a72b0961ed3e.txt\n\nAre you sure your test is doing the right thing? ","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"40cbb438cbed494a2c49565b10ca1ab7140b93a1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"13aab658_0c7ee5c4","updated":"2021-11-12 07:58:57.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"5d0e5be8324690cd95abe676f0e4b245c1068b2d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"2f0f7fbb_55c286ee","updated":"2021-11-23 01:42:14.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"298e5c8937f3997a8232228b2bde826fc2e5395c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"46bb16fe_eb93f9f4","updated":"2021-11-23 01:41:25.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"0d9d6033fc3cbc74a95da4d99378064463499c9d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"47ebd3bf_c1740487","updated":"2021-11-17 08:58:57.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"93d7a42067859b045047915bd3b6131fa6ae126d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"5aeb40bd_bd5d897f","updated":"2021-11-12 01:25:40.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"940add26983e093b25471d918554b5352dac6d7c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"6c6f770b_5fe0be97","updated":"2021-11-10 03:30:21.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"ab7bb249372231dd90f5462463831fc404a3cf01","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"74163fcd_71efe5a1","updated":"2021-11-14 11:42:09.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"73a87e741a67e52cdf437d9f7ea41ed6a51e32e1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"990b77a3_4843221f","updated":"2021-11-09 11:24:01.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"db7d270d841ac4429af90e32659f35902f0dd7d7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"9f6029a9_aa771352","updated":"2021-11-15 02:49:48.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"bac431c338102acfcde03652d3e89d81bbfad53d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"e9f43e93_7c6d89cb","updated":"2021-11-11 10:24:32.000000000","message":"charm-recheck","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"52650d5b7cde5b43cab45d847014db969461a8c1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"c75e54e6_dbad64b5","updated":"2021-11-15 02:49:34.000000000","message":"it\u0027s still timeout issue so re-trigger again","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"846b81bbb51f0fdee5b2fa21ca3f3e853f53237f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"53e031b1_df85de87","updated":"2021-11-12 07:58:44.000000000","message":"it\u0027s timeout issue so I will retrigger again\n\n2021-11-12 06:35:21.070380 | bionic-medium | 2021-11-12 06:35:21 [INFO] TIMEOUT: Workloads didn\u0027t reach acceptable status:\n2021-11-12 06:35:21.070910 | bionic-medium | 2021-11-12 06:35:21 [INFO] Timed out waiting for \u0027octavia/3\u0027. The unit status is \u0027not idle\u0027 which is not one of \u0027[\u0027idle\u0027]\u0027","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"6e1b290187478ddc337fa2f8b3ebdcc2bdac245a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"8675a893_662f5d03","updated":"2021-11-10 03:30:15.000000000","message":"one day has passwed, don\u0027t know why charm-recheck didn\u0027t retrigger ci, so retrigger again now","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"7b8fdc5cc37e85cbd97adcd3ea165d8e3e3bcd00","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"e6642f5e_6068af6b","updated":"2021-11-17 08:58:45.000000000","message":"so let us retrigger to see what happens ","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"fb96c1e1c875023a7d6e5de8b0e8659cca010b08","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"2d4ddfbb_d8a67b0f","updated":"2021-11-14 11:41:57.000000000","message":"still timeout issue, so retrigger again\n\n Timed out waiting for \u0027octavia/3\u0027. The unit status is \u0027not idle\u0027 which is not one of \u0027[\u0027idle\u0027]\u0027","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"fec4b04dcb8486f23822145ea91ea2ad98c5c28f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"31b43cd7_8e89afd0","in_reply_to":"2d491fdd_85c657f7","updated":"2021-11-18 08:49:37.000000000","message":"Hi Aurelien, pls see the following two comments, and I also created another bug - https://bugs.launchpad.net/charm-hacluster/+bug/1951392","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"298e5c8937f3997a8232228b2bde826fc2e5395c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"488d80c6_5f69ceaf","in_reply_to":"6338b3a6_e39cfc6c","updated":"2021-11-23 01:41:25.000000000","message":"Hi Aurelien, thank you so much!","commit_id":"cc292d72e2cb47366e093490c5319503f86d93ec"}],"src/lib/charm/openstack/api_crud.py":[{"author":{"_account_id":10366,"name":"Hemanth N","email":"hemanth.nakkina@canonical.com","username":"Hemanth"},"change_message_id":"17a3e3b307615806368a489b2bddaab6f4b775a8","unresolved":true,"context_lines":[{"line_number":266,"context_line":"    session \u003d session_from_identity_service(identity_service)"},{"line_number":267,"context_line":"    try:"},{"line_number":268,"context_line":"        nc \u003d init_neutron_client(session)"},{"line_number":269,"context_line":"        port \u003d lookup_hm_port(nc, local_unit_name)"},{"line_number":270,"context_line":"        nc.delete_port(port[\u0027id\u0027])"},{"line_number":271,"context_line":"    except NEUTRON_TEMP_EXCS as e:"},{"line_number":272,"context_line":"        raise APIUnavailable(\u0027neutron\u0027, \u0027ports\u0027, e)"}],"source_content_type":"text/x-python","patch_set":7,"id":"75f748e4_4eb09c84","line":269,"range":{"start_line":269,"start_character":15,"end_line":269,"end_character":29},"updated":"2021-06-29 11:52:53.000000000","message":"lookup_hm_port can return port or DuplicateResource or None.\nJust for completeness, can you handle the case where lookup_hm_port returns None.","commit_id":"1ab561fd2621b47bbf453fefa1a2889cf2aff505"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"fd7faf6009fba4758918d4d96395f32683d0cfac","unresolved":false,"context_lines":[{"line_number":266,"context_line":"    session \u003d session_from_identity_service(identity_service)"},{"line_number":267,"context_line":"    try:"},{"line_number":268,"context_line":"        nc \u003d init_neutron_client(session)"},{"line_number":269,"context_line":"        port \u003d lookup_hm_port(nc, local_unit_name)"},{"line_number":270,"context_line":"        nc.delete_port(port[\u0027id\u0027])"},{"line_number":271,"context_line":"    except NEUTRON_TEMP_EXCS as e:"},{"line_number":272,"context_line":"        raise APIUnavailable(\u0027neutron\u0027, \u0027ports\u0027, e)"}],"source_content_type":"text/x-python","patch_set":7,"id":"b9b1fb02_db029d9e","line":269,"range":{"start_line":269,"start_character":15,"end_line":269,"end_character":29},"in_reply_to":"75f748e4_4eb09c84","updated":"2021-07-01 01:49:55.000000000","message":"Done","commit_id":"1ab561fd2621b47bbf453fefa1a2889cf2aff505"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"bda2aeba7f0ae6510f394aee9b12f4726e1d65e1","unresolved":true,"context_lines":[{"line_number":575,"context_line":""},{"line_number":576,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":577,"context_line":"    :type identity_service: RelationBase class"},{"line_number":578,"context_line":"    :returns: List of IP addresses extracted from port details in search result"},{"line_number":579,"context_line":"    :rtype: list of str"},{"line_number":580,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":581,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"faa27dc5_18141ea4","line":578,"updated":"2021-07-02 09:35:01.000000000","message":"we\u0027re now returning two things\n\nAnd it seems like the second structure (the map) contains more info than the first structure (the list). So is it really necessary still to return the list? There is only one single caller to this function: could we just return the map and adapt the caller to make use of the map only? The caller can convert the map back into a list where needed, e.g. with `my_map.values()` or with a list comprehension.","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"f508c6be0b4bb4a22d31a1db4b067b48b9f43fe9","unresolved":true,"context_lines":[{"line_number":575,"context_line":""},{"line_number":576,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":577,"context_line":"    :type identity_service: RelationBase class"},{"line_number":578,"context_line":"    :returns: List of IP addresses extracted from port details in search result"},{"line_number":579,"context_line":"    :rtype: list of str"},{"line_number":580,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":581,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"715bab49_55aa157e","line":578,"in_reply_to":"faa27dc5_18141ea4","updated":"2021-07-05 04:23:41.000000000","message":"Done, but I have to change the function name from get_port_ips to get_port_ip_unit_map as well since we have removed the orignal neutron_ip_list","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"bda2aeba7f0ae6510f394aee9b12f4726e1d65e1","unresolved":true,"context_lines":[{"line_number":576,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":577,"context_line":"    :type identity_service: RelationBase class"},{"line_number":578,"context_line":"    :returns: List of IP addresses extracted from port details in search result"},{"line_number":579,"context_line":"    :rtype: list of str"},{"line_number":580,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":581,"context_line":"    \"\"\""},{"line_number":582,"context_line":"    session \u003d session_from_identity_service(identity_service)"}],"source_content_type":"text/x-python","patch_set":9,"id":"312ccf43_8b89bfa0","line":579,"updated":"2021-07-02 09:35:01.000000000","message":"The return type has changed","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"f508c6be0b4bb4a22d31a1db4b067b48b9f43fe9","unresolved":false,"context_lines":[{"line_number":576,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":577,"context_line":"    :type identity_service: RelationBase class"},{"line_number":578,"context_line":"    :returns: List of IP addresses extracted from port details in search result"},{"line_number":579,"context_line":"    :rtype: list of str"},{"line_number":580,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":581,"context_line":"    \"\"\""},{"line_number":582,"context_line":"    session \u003d session_from_identity_service(identity_service)"}],"source_content_type":"text/x-python","patch_set":9,"id":"8ce1611b_8f99bd04","line":579,"in_reply_to":"312ccf43_8b89bfa0","updated":"2021-07-05 04:23:41.000000000","message":"Done","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c8226549eef65c61a09caf97d61d39ab2e637c1f","unresolved":true,"context_lines":[{"line_number":255,"context_line":"def delete_hm_port(identity_service, local_unit_name):"},{"line_number":256,"context_line":"    \"\"\"Delete port object for Octavia hm port for local unit."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"    :param nc: Neutron Client object"},{"line_number":259,"context_line":"    :type nc: neutron_client.Client"},{"line_number":260,"context_line":"    :param local_unit_name: Name of juju unit, used to build tag name for port"},{"line_number":261,"context_line":"    :type local_unit_name: str"}],"source_content_type":"text/x-python","patch_set":27,"id":"bf51da47_df6a2e8a","line":258,"updated":"2021-11-04 09:46:10.000000000","message":"There is no such param","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a09bd9eafac67d25b52e2c1071b264203ebdb63c","unresolved":false,"context_lines":[{"line_number":255,"context_line":"def delete_hm_port(identity_service, local_unit_name):"},{"line_number":256,"context_line":"    \"\"\"Delete port object for Octavia hm port for local unit."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"    :param nc: Neutron Client object"},{"line_number":259,"context_line":"    :type nc: neutron_client.Client"},{"line_number":260,"context_line":"    :param local_unit_name: Name of juju unit, used to build tag name for port"},{"line_number":261,"context_line":"    :type local_unit_name: str"}],"source_content_type":"text/x-python","patch_set":27,"id":"aa08a031_0f62d9df","line":258,"in_reply_to":"bf51da47_df6a2e8a","updated":"2021-11-08 04:10:28.000000000","message":"Done, changed to:\n\n    :param identity_service: reactive Endpoint of type ``identity-service``     \n    :type identity_service: RelationBase class","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c8226549eef65c61a09caf97d61d39ab2e637c1f","unresolved":true,"context_lines":[{"line_number":260,"context_line":"    :param local_unit_name: Name of juju unit, used to build tag name for port"},{"line_number":261,"context_line":"    :type local_unit_name: str"},{"line_number":262,"context_line":"    :returns: None"},{"line_number":263,"context_line":"    :raises: DuplicateResource or any exceptions raised by Keystone and Neutron"},{"line_number":264,"context_line":"             clients."},{"line_number":265,"context_line":"    \"\"\""},{"line_number":266,"context_line":"    session \u003d session_from_identity_service(identity_service)"}],"source_content_type":"text/x-python","patch_set":27,"id":"ed0ce4dd_9d33f4fa","line":263,"updated":"2021-11-04 09:46:10.000000000","message":"It looks like you also raise APIUnavailable instead of some of the Keystone or Neutron exceptions.","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a09bd9eafac67d25b52e2c1071b264203ebdb63c","unresolved":false,"context_lines":[{"line_number":260,"context_line":"    :param local_unit_name: Name of juju unit, used to build tag name for port"},{"line_number":261,"context_line":"    :type local_unit_name: str"},{"line_number":262,"context_line":"    :returns: None"},{"line_number":263,"context_line":"    :raises: DuplicateResource or any exceptions raised by Keystone and Neutron"},{"line_number":264,"context_line":"             clients."},{"line_number":265,"context_line":"    \"\"\""},{"line_number":266,"context_line":"    session \u003d session_from_identity_service(identity_service)"}],"source_content_type":"text/x-python","patch_set":27,"id":"c6c84b0d_e0e882ec","line":263,"in_reply_to":"ed0ce4dd_9d33f4fa","updated":"2021-11-08 04:10:28.000000000","message":"change to:\n\n:raises: api_crud.APIUnavailable","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c8226549eef65c61a09caf97d61d39ab2e637c1f","unresolved":true,"context_lines":[{"line_number":578,"context_line":""},{"line_number":579,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":580,"context_line":"    :type identity_service: RelationBase class"},{"line_number":581,"context_line":"    :returns: Map of IP and Unit extracted from port details in search result"},{"line_number":582,"context_line":"    :rtype: dict(str, str)"},{"line_number":583,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":584,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":27,"id":"748066ec_696fce23","line":581,"updated":"2021-11-04 09:46:10.000000000","message":"Should be \"Map of unit names and ip addresses\"","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a09bd9eafac67d25b52e2c1071b264203ebdb63c","unresolved":false,"context_lines":[{"line_number":578,"context_line":""},{"line_number":579,"context_line":"    :param identity_service: reactive Endpoint of type ``identity-service``"},{"line_number":580,"context_line":"    :type identity_service: RelationBase class"},{"line_number":581,"context_line":"    :returns: Map of IP and Unit extracted from port details in search result"},{"line_number":582,"context_line":"    :rtype: dict(str, str)"},{"line_number":583,"context_line":"    :raises: api_crud.APIUnavailable"},{"line_number":584,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":27,"id":"b0a60630_0a9e4035","line":581,"in_reply_to":"748066ec_696fce23","updated":"2021-11-08 04:10:28.000000000","message":"Done","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"}],"src/reactive/octavia_handlers.py":[{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"bda2aeba7f0ae6510f394aee9b12f4726e1d65e1","unresolved":true,"context_lines":[{"line_number":204,"context_line":"            removing_ip \u003d neutron_ip_unit_map.get(departing_unit, None)"},{"line_number":205,"context_line":"            ch_core.hookenv.log(\u0027removing neutron_ip_list: {}\u0027.format("},{"line_number":206,"context_line":"                removing_ip), level\u003dch_core.hookenv.DEBUG)"},{"line_number":207,"context_line":"            if removing_ip in neutron_ip_list:"},{"line_number":208,"context_line":"                neutron_ip_list.remove(removing_ip)"},{"line_number":209,"context_line":"                try:"},{"line_number":210,"context_line":"                    ch_core.hookenv.log(\u0027deleting hm port for departed \u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"041e6595_8b46445c","line":207,"updated":"2021-07-02 09:35:01.000000000","message":"can this ever be False? anything that is in neutron_ip_list is also in neutron_ip_unit_map.","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"f508c6be0b4bb4a22d31a1db4b067b48b9f43fe9","unresolved":true,"context_lines":[{"line_number":204,"context_line":"            removing_ip \u003d neutron_ip_unit_map.get(departing_unit, None)"},{"line_number":205,"context_line":"            ch_core.hookenv.log(\u0027removing neutron_ip_list: {}\u0027.format("},{"line_number":206,"context_line":"                removing_ip), level\u003dch_core.hookenv.DEBUG)"},{"line_number":207,"context_line":"            if removing_ip in neutron_ip_list:"},{"line_number":208,"context_line":"                neutron_ip_list.remove(removing_ip)"},{"line_number":209,"context_line":"                try:"},{"line_number":210,"context_line":"                    ch_core.hookenv.log(\u0027deleting hm port for departed \u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"ea0c0609_09fb3b30","line":207,"in_reply_to":"041e6595_8b46445c","updated":"2021-07-05 04:23:41.000000000","message":"now I am using \u0027neutron_ip_list \u003d sorted(neutron_ip_unit_map.values())\u0027 you suggested so there is no case that is in neutron_ip_list and also in neutron_ip_unit_map. Besides, this can be ever be False when update_controller_ip_port_list is triggered many times, so it will be defensive","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a32ec3a8d6abf61a1bc69baa4e6c0a7f0c1509ab","unresolved":false,"context_lines":[{"line_number":204,"context_line":"            removing_ip \u003d neutron_ip_unit_map.get(departing_unit, None)"},{"line_number":205,"context_line":"            ch_core.hookenv.log(\u0027removing neutron_ip_list: {}\u0027.format("},{"line_number":206,"context_line":"                removing_ip), level\u003dch_core.hookenv.DEBUG)"},{"line_number":207,"context_line":"            if removing_ip in neutron_ip_list:"},{"line_number":208,"context_line":"                neutron_ip_list.remove(removing_ip)"},{"line_number":209,"context_line":"                try:"},{"line_number":210,"context_line":"                    ch_core.hookenv.log(\u0027deleting hm port for departed \u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"83169fa7_9a3fdd62","line":207,"in_reply_to":"159facd9_409dbd71","updated":"2021-07-07 06:23:09.000000000","message":"Done","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"1d3f26b77081db3544cdfe042acd7eacd8f89171","unresolved":true,"context_lines":[{"line_number":204,"context_line":"            removing_ip \u003d neutron_ip_unit_map.get(departing_unit, None)"},{"line_number":205,"context_line":"            ch_core.hookenv.log(\u0027removing neutron_ip_list: {}\u0027.format("},{"line_number":206,"context_line":"                removing_ip), level\u003dch_core.hookenv.DEBUG)"},{"line_number":207,"context_line":"            if removing_ip in neutron_ip_list:"},{"line_number":208,"context_line":"                neutron_ip_list.remove(removing_ip)"},{"line_number":209,"context_line":"                try:"},{"line_number":210,"context_line":"                    ch_core.hookenv.log(\u0027deleting hm port for departed \u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"159facd9_409dbd71","line":207,"in_reply_to":"ea0c0609_09fb3b30","updated":"2021-07-05 08:48:51.000000000","message":"```\nneutron_ip_list \u003d sorted(neutron_ip_unit_map.values())\n[...]\nremoving_ip \u003d neutron_ip_unit_map.get(departing_unit, None)\n```\n\nSo it is \"mathematically\" not possible to find removing_ip in the list but not in the map here since the list is generated from the map. Does that make sense?\n\nThere is no need to defend against something mathematically impossible :) The real thing you should be testing here is `if removing_ip is not None:`.","commit_id":"4e57b9418b907705849212c5854095e9f8d07be0"},{"author":{"_account_id":31289,"name":"Aurelien Lourot","email":"aurelien.lourot@gmail.com","username":"lourot"},"change_message_id":"c8226549eef65c61a09caf97d61d39ab2e637c1f","unresolved":true,"context_lines":[{"line_number":231,"context_line":"                    api_crud.delete_hm_port(identity_service, departing_unit)"},{"line_number":232,"context_line":"                except api_crud.APIUnavailable as e:"},{"line_number":233,"context_line":"                    ch_core.hookenv.log(\u0027Neutron API not available yet,\u0027"},{"line_number":234,"context_line":"                                        \u0027deferring port discovery. (\"{}\")\u0027"},{"line_number":235,"context_line":"                                        .format(e),"},{"line_number":236,"context_line":"                                        level\u003dch_core.hookenv.DEBUG)"},{"line_number":237,"context_line":"                    return"}],"source_content_type":"text/x-python","patch_set":27,"id":"02c315a4_4805b478","line":234,"updated":"2021-11-04 09:46:10.000000000","message":"\"port discovery\"? This looks like a copy-paste mistake.","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"},{"author":{"_account_id":2711,"name":"Zhang Hua","display_name":"Zhang Hua","email":"joshua.zhang@canonical.com","username":"zhhuabj"},"change_message_id":"a09bd9eafac67d25b52e2c1071b264203ebdb63c","unresolved":false,"context_lines":[{"line_number":231,"context_line":"                    api_crud.delete_hm_port(identity_service, departing_unit)"},{"line_number":232,"context_line":"                except api_crud.APIUnavailable as e:"},{"line_number":233,"context_line":"                    ch_core.hookenv.log(\u0027Neutron API not available yet,\u0027"},{"line_number":234,"context_line":"                                        \u0027deferring port discovery. (\"{}\")\u0027"},{"line_number":235,"context_line":"                                        .format(e),"},{"line_number":236,"context_line":"                                        level\u003dch_core.hookenv.DEBUG)"},{"line_number":237,"context_line":"                    return"}],"source_content_type":"text/x-python","patch_set":27,"id":"7464856e_f886b76c","line":234,"in_reply_to":"02c315a4_4805b478","updated":"2021-11-08 04:10:28.000000000","message":"change to \u0027deferring hm port deletion\u0027","commit_id":"c073b1a7ba4842c82ce29acdf7856707403b4abc"}]}