)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"334dde8411d3a8a44401235cdf313d4edfe190d0","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[OVN] Do not fail if the ACL does not exists in the deletion"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Depends-On: https://review.opendev.org/c/openstack/ovsdbapp/+/934407"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Closes-Bug: #2084977"},{"line_number":12,"context_line":"Change-Id: I179d859bbcb0a8a2b3575bc03d703ebb0a6dc0a9"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9a86597f_ce726639","line":9,"updated":"2024-11-19 22:22:49.000000000","message":"should probably be WIP until we can bump the library version?","commit_id":"d4ac876a3c20151ba6a10c58e5be2d69f2edbc8d"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"42c8e86fc61e69ec45a662c0cdf939ea4fe3aa4e","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[OVN] Do not fail if the ACL does not exists in the deletion"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Depends-On: https://review.opendev.org/c/openstack/ovsdbapp/+/934407"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Closes-Bug: #2084977"},{"line_number":12,"context_line":"Change-Id: I179d859bbcb0a8a2b3575bc03d703ebb0a6dc0a9"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"96f7aca3_f4ca2527","line":9,"in_reply_to":"9a86597f_ce726639","updated":"2024-11-20 10:31:41.000000000","message":"It will fail in any case so it won\u0027t be possible to merge it","commit_id":"d4ac876a3c20151ba6a10c58e5be2d69f2edbc8d"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"26253c808c47be43646947cd6c6e19fc49e9eed7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"eea1dd30_431183f7","updated":"2024-12-04 11:54:34.000000000","message":"recheck neutron-functional","commit_id":"67297a33d7cb24156694adb029e1223e7a7c7ccd"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"7ea11016b9ccf72cf0fca792d7fc6a3904f5b46e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"78b2651e_8dd9aa69","updated":"2024-12-09 05:50:48.000000000","message":"This looks will just hide the issue as seen below ovndb have the record, ex job:-\nhttps://c15d161aa166386e5680-f95fc25bff3a61ae232d3b95f165c0d8.ssl.cf1.rackcdn.com/936272/4/check/neutron-ovn-tempest-ipv6-only-ovs-release-wsgi-11/c507430/job-output.txt running without this patch\nfailed as:- 2024-12-07 13:12:32.523237 | controller | Details: {\u0027type\u0027: \u0027SecurityGroupInUse\u0027, \u0027message\u0027: \u0027Security Group None cannot perform precommit_delete due to Callback neutron.plugins.ml2.drivers.ovn.mech_driver.mech_driver.OVNMechanismDriver._delete_security_group_precommit-1754321 failed with \"Port_Group pg_1f06489c_c345_41f8_8da4_4ce9d255bd30 does not exist\".\u0027, \u0027detail\u0027: \u0027\u0027}\n\nBut db have the port group\novsdb-tool transact ovnnb_db.db \u0027[\"OVN_Northbound\", {\"op\": \"select\", \"table\": \"Port_Group\", \"where\": [[\"name\", \"\u003d\u003d\", \"pg_1f06489c_c345_41f8_8da4_4ce9d255bd30\"]]}]\u0027 |jq .[0].rows[].name\n\"pg_1f06489c_c345_41f8_8da4_4ce9d255bd30\"\n\nneutron api\nDec 07 13:12:32.312566 np0039284412 devstack@neutron-api.service[61683]: DEBUG neutron.common.ovn.hash_ring_manager [-] Hash Ring loaded. 2 active nodes. 3 offline nodes {{(pid\u003d61683) _load_hash_ring /opt/stack/neutron/neutron/common/ovn/hash_ring_manager.py:102}}\nDec 07 13:12:32.322027 np0039284412 devstack@neutron-api.service[61683]: WARNING ovsdbapp.backend.ovs_idl.vlog [-] ssl:[2607:5300:201:2000::154]:6641: receive error: Transport endpoint is not connected\nDec 07 13:12:32.326324 np0039284412 devstack@neutron-api.service[61683]: WARNING ovsdbapp.backend.ovs_idl.vlog [-] ssl:[2607:5300:201:2000::154]:6641: connection dropped (Transport endpoint is not connected)\nDec 07 13:12:32.327734 np0039284412 devstack@neutron-api.service[61683]: DEBUG futurist.periodics [-] Submitting periodic callback \u0027neutron.plugins.ml2.drivers.ovn.mech_driver.ovsdb.maintenance.HashRingHealthCheckPeriodics.touch_hash_ring_node\u0027 {{(pid\u003d61683) _process_scheduled /opt/stack/data/venv/lib/python3.12/site-packages/futurist/periodics.py:638}}\nDec 07 13:12:32.331374 np0039284412 devstack@neutron-api.service[61683]: DEBUG neutron.common.ovn.hash_ring_manager [-] Hash Ring loaded. 4 active nodes. 3 offline nodes {{(pid\u003d61683) _load_hash_ring /opt/stack/neutron/neutron/common/ovn/hash_ring_manager.py:102}}\nDec 07 13:12:32.372058 np0039284412 devstack@neutron-api.service[61683]: DEBUG neutron_lib.callbacks.manager [None req-d23ae1b6-834f-4653-92e4-f344990c1756 admin admin] Publish callbacks [] for security_group (1f06489c-c345-41f8-8da4-4ce9d255bd30), before_delete {{(pid\u003d61683) _notify_loop /opt/stack/data/venv/lib/python3.12/site-packages/neutron_lib/callbacks/manager.py:184}}\nDec 07 13:12:32.389299 np0039284412 devstack@neutron-api.service[61683]: DEBUG neutron_lib.callbacks.manager [None req-d23ae1b6-834f-4653-92e4-f344990c1756 admin admin] Publish callbacks [\u0027neutron.plugins.ml2.drivers.ovn.mech_driver.mech_driver.OVNMechanismDriver._delete_security_group_precommit-1754321\u0027] for security_group (1f06489c-c345-41f8-8da4-4ce9d255bd30), precommit_delete {{(pid\u003d61683) _notify_loop /opt/stack/data/venv/lib/python3.12/site-packages/neutron_lib/callbacks/manager.py:184}}\nDec 07 13:12:32.432628 np0039284412 devstack@neutron-api.service[61683]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n\u003d1 command(idx\u003d0): PgAclDelCommand(_result\u003dNone, entity\u003dpg_1f06489c_c345_41f8_8da4_4ce9d255bd30, if_exists\u003dFalse, conditions\u003d[(\u0027direction\u0027, \u0027\u003d\u0027, \u0027to-lport\u0027), (\u0027priority\u0027, \u0027\u003d\u0027, 1002), (\u0027match\u0027, \u0027\u003d\u0027, \u0027outport \u003d\u003d @pg_1f06489c_c345_41f8_8da4_4ce9d255bd30 \u0026\u0026 ip6 \u0026\u0026 ip6.src \u003d\u003d $pg_1f06489c_c345_41f8_8da4_4ce9d255bd30_ip6\u0027)]) {{(pid\u003d61683) do_commit /opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/transaction.py:89}}\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]: ERROR ovsdbapp.backend.ovs_idl.transaction [None req-d23ae1b6-834f-4653-92e4-f344990c1756 admin admin] Traceback (most recent call last):\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/schema/ovn_northbound/commands.py\", line 211, in run_idl\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     entity \u003d self.api.lookup(self.lookup_table, self.entity)\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/__init__.py\", line 184, in lookup\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     return self._lookup(table, record)\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/__init__.py\", line 235, in _lookup\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     row \u003d idlutils.row_by_value(self, rl.table, rl.column, record)\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/idlutils.py\", line 114, in row_by_value\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     raise RowNotFound(table\u003dtable, col\u003dcolumn, match\u003dmatch)\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]: ovsdbapp.backend.ovs_idl.idlutils.RowNotFound: Cannot find Port_Group with name\u003dpg_1f06489c_c345_41f8_8da4_4ce9d255bd30\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]: The above exception was the direct cause of the following exception:\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]: Traceback (most recent call last):\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/connection.py\", line 118, in run\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     txn.results.put(txn.do_commit())\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:                     ^^^^^^^^^^^^^^^\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/backend/ovs_idl/transaction.py\", line 92, in do_commit\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     command.run_idl(txn)\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:   File \"/opt/stack/data/venv/lib/python3.12/site-packages/ovsdbapp/schema/ovn_northbound/commands.py\", line 216, in run_idl\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]:     raise RuntimeError(msg) from e\nDec 07 13:12:32.436517 np0039284412 devstack@neutron-api.service[61683]: RuntimeError: Port_Group pg_1f06489c_c345_41f8_8da4_4ce9d255bd30 does not exist","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"4ee09473cba771c357d5e8326ed94a39eba7f8b8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"c90e7d93_83122c99","in_reply_to":"0b8cfe6a_6269fa07","updated":"2024-12-09 10:19:00.000000000","message":"Actually this is not an error but how the CI job is configured. The number of detected nodes is correct (but the neutron.conf file doesn\u0027t have the correct number). I\u0027ll continue investigating.","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"c4ae19abfe2985ebb83842b3d8b1016816e47d15","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"728c8a68_ed4b3b39","in_reply_to":"36051ea1_c2401748","updated":"2025-01-16 13:42:41.000000000","message":"That was a mistake, the upper LP is not the cause of this issue. Is still happening with the patch https://review.opendev.org/c/openstack/neutron/+/937351","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"2e59edbe2b044e4445b719a187f5aa5c65fc1f6c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"b879fa59_00501132","in_reply_to":"728c8a68_ed4b3b39","updated":"2025-01-16 15:33:00.000000000","message":"As commented in IRC, I\u0027ve recently see that in the CI jobs of https://review.opendev.org/c/openstack/neutron/+/939347/4.\n\nFor example, 2 out of 20 jobs:\n* https://zuul.opendev.org/t/openstack/build/68d97de5b9914c3e947b922fc257a061\n* https://zuul.opendev.org/t/openstack/build/4f15b08d8e4b4273b6cffc8994287f32","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"fa25c2764b9b466fd594732a187999465932d222","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"0b8cfe6a_6269fa07","in_reply_to":"78b2651e_8dd9aa69","updated":"2024-12-09 10:08:07.000000000","message":"In this same log we can see this:\nDec 07 13:12:32.327248 np0039284412 devstack@neutron-api.service[61683]: CRITICAL neutron.plugins.ml2.drivers.ovn.mech_driver.ovsdb.maintenance [-] The number of nodes in the Hash Ring (5) is higher than the number of API workers (1) for host \"np0039284412\". Something is not right and OVSDB events could be missed because of this. Please check the status of the Neutron processes, this can happen when the API workers are killed and restarted. Restarting the service should fix the issue, see LP #2024205 for more information.\n\nThere is a previous issue that should be addressed here. I should improve https://review.opendev.org/c/openstack/neutron/+/936428/ first.\n\nActually because the hash ring is in error state, the PG deletion event is processed by other node while it should be processed by the same one that created it, because ``OvnIdlDistributedLock`` and the ``HashRingManager`` should assign the same node.","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"93c6898a45c153fa2add096eee7f96096d7998ef","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"36051ea1_c2401748","in_reply_to":"a7c475ab_52af84d1","updated":"2025-01-16 13:39:27.000000000","message":"So if root cause is being fixed why we really need this patch? as seen above it can leave stale data and hide actual issue.\nor you mean the issue is seen in eventlet jobs too? can you share the log link if that\u0027s the case.","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"8b43195df924770e90456e45c6798c0966a1c186","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"f95992f3_78891781","in_reply_to":"b879fa59_00501132","updated":"2025-01-17 14:04:55.000000000","message":"Ok checked both and i see this patch will hide the actual issue.\nhttps://zuul.opendev.org/t/openstack/build/68d97de5b9914c3e947b922fc257a061 reports 2025-01-16 14:58:06.623869 | controller | Details: {\u0027type\u0027: \u0027SecurityGroupInUse\u0027, \u0027message\u0027: \u0027Security Group None cannot perform precommit_delete due to Callback neutron.plugins.ml2.drivers.ovn.mech_driver.mech_driver.OVNMechanismDriver._delete_security_group_precommit-980715 failed with \"Port_Group pg_defdf4fa_160a_4ea6_b95f_5caed8b6f628 does not exist\".\u0027, \u0027detail\u0027: \u0027\u0027} when deleting the security group\nbut looking at db port group exist\n$ sudo ovn-nbctl list port_group pg_defdf4fa_160a_4ea6_b95f_5caed8b6f628\n_uuid               : dd6ba1ff-03a0-405e-b744-191186a457d5\nacls                : [da7a9f73-2734-4525-879e-671de43df59e, dcb4696e-8e21-46e8-8222-59b010f2473d, df566bd1-3784-4013-a9ea-b325aaebb79a, ead3624c-edd7-4643-aeb4-e26fb4293863]\nexternal_ids        : {\"neutron:security_group_id\"\u003d\"defdf4fa-160a-4ea6-b95f-5caed8b6f628\"}\nname                : pg_defdf4fa_160a_4ea6_b95f_5caed8b6f628\nports               : []\nSo with this patch the request will not fail and will leave stale data in ovn.\n\nSimilarly in https://zuul.opendev.org/t/openstack/build/4f15b08d8e4b4273b6cffc8994287f32 reports 2025-01-16 14:58:34.228561 | controller | Details: {\u0027type\u0027: \u0027SecurityGroupInUse\u0027, \u0027message\u0027: \u0027Security Group None cannot perform precommit_delete due to Callback neutron.plugins.ml2.drivers.ovn.mech_driver.mech_driver.OVNMechanismDriver._delete_security_group_precommit-436679 failed with \"Port_Group pg_999b2844_28e1_49bd_93da_59ed3909f4ee does not exist\".\u0027, \u0027detail\u0027: \u0027\u0027} when deleting security group and ovn db have the data\n$ sudo ovn-nbctl list port_group pg_999b2844_28e1_49bd_93da_59ed3909f4ee                                                                                                                                        \n_uuid               : 0e0b7b4e-241a-46d2-bb09-c46fd1019e90\nacls                : [0f419359-f2f6-4497-a8cb-214b656598a3, 200a9359-a2e3-4f6d-bd7e-1b1249ac0506, 7fc83790-adf2-4e09-8260-02fc72fe576e, f7f5b281-8bae-4b0c-ac70-945a7a1b5282]\nexternal_ids        : {\"neutron:security_group_id\"\u003d\"999b2844-28e1-49bd-93da-59ed3909f4ee\"}\nname                : pg_999b2844_28e1_49bd_93da_59ed3909f4ee\nports               : []","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"3acc7ab5f66043b3a8837ad0e6929aa977de8983","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"a7c475ab_52af84d1","in_reply_to":"c90e7d93_83122c99","updated":"2024-12-09 11:56:48.000000000","message":"I\u0027ve created https://bugs.launchpad.net/neutron/+bug/2091284 and pushed a new PS.\n\nI\u0027ve also updated the testing patch pointing to this new patch.","commit_id":"7e6a3104e8c8735d7bb5a2f98d7dd7bc61afc191"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"882fca244474638938df4356977ba7352568e4ec","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"972d23e2_ccac095b","updated":"2024-12-10 06:59:55.000000000","message":"recheck neutron-functional","commit_id":"716f4d4f84bffc1e61cc1d8bb7b6aa52ade1c244"}]}