)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"99ff6680bb05291bb28664757012e2c64a0c9e6a","unresolved":true,"context_lines":[{"line_number":17,"context_line":"Closes-Bug: #2077152"},{"line_number":18,"context_line":"Change-Id: Ib04346eef7e88d01e6c6bda79e10708cac553321"},{"line_number":19,"context_line":"Signed-off-by: lajoskatona \u003clajos.katona@est.tech\u003e"},{"line_number":20,"context_line":"Signed-off-by: Nikita Kushnarenko \u003cNKushnarenko@itkey.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"0f363277_0be9cd21","line":20,"range":{"start_line":20,"start_character":0,"end_line":20,"end_character":58},"updated":"2026-04-30 11:51:41.000000000","message":"after discussing with colleagues, removing this signed-off-by","commit_id":"7f4b6c6f973a8f5f3e843c2807b104f9561d174f"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"8685ed2094efec5feb92130d14fed855f5e486db","unresolved":false,"context_lines":[{"line_number":17,"context_line":"Closes-Bug: #2077152"},{"line_number":18,"context_line":"Change-Id: Ib04346eef7e88d01e6c6bda79e10708cac553321"},{"line_number":19,"context_line":"Signed-off-by: lajoskatona \u003clajos.katona@est.tech\u003e"},{"line_number":20,"context_line":"Signed-off-by: Nikita Kushnarenko \u003cNKushnarenko@itkey.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"1f35d3a0_748c0605","line":20,"range":{"start_line":20,"start_character":0,"end_line":20,"end_character":58},"in_reply_to":"0f363277_0be9cd21","updated":"2026-06-15 17:04:27.000000000","message":"Done","commit_id":"7f4b6c6f973a8f5f3e843c2807b104f9561d174f"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":36083,"name":"Nikita Kushnarenko","display_name":"Nikita Kushnarenko","email":"NKushnarenko@itkey.com","username":"NickKush"},"change_message_id":"e11aaa3fd65267a6474ffa1cc8be7e949ca73ee5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"a8e2f499_a0dc91f7","updated":"2024-12-05 12:47:15.000000000","message":"Zuul raised 2 errors that are most likely unrelated to the patch. \nI\u0027m not sure what to do with them\n\ntempest-full-py3:\n```\n+ lib/horizon:init_horizon:167             :   django-admin compress --force\n/opt/stack/data/venv/lib/python3.12/site-packages/django/conf/__init__.py:267: RemovedInDjango50Warning: The USE_L10N setting is deprecated. Starting with Django 5.0, localized formatting of data will always be enabled. For example Django will display numbers and dates using the format of the current locale.\n  warnings.warn(USE_L10N_DEPRECATED_MSG, RemovedInDjango50Warning)\n/opt/stack/data/venv/lib/python3.12/site-packages/debreach/__init__.py:6: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n  version_info \u003d version.StrictVersion(__version__).version\nCompressing... CommandError: An error occurred during rendering serial_console.html: Syntax error: Found \u0027inline-blo\u0027 but expected one of ADD, ALPHA_FUNCTION, BANG_IMPORTANT, BAREWORD, COLOR, DOUBLE_QUOTE, FNCT, IF_FUNCTION, INTERP_START, LITERAL_FUNCTION, LPAR, NOT, NUM, SIGN, SINGLE_QUOTE, URL_FUNCTION, VAR\n+ lib/horizon:init_horizon:1               :   exit_trap\n+ ./stack.sh:exit_trap:543                 :   local r\u003d1\n++ ./stack.sh:exit_trap:544                 :   jobs -p\n+ ./stack.sh:exit_trap:544                 :   jobs\u003d46825\n+ ./stack.sh:exit_trap:547                 :   [[ -n 46825 ]]\n+ ./stack.sh:exit_trap:547                 :   [[ -n /opt/stack/logs/devstacklog.txt.2024-12-03-084734 ]]\n+ ./stack.sh:exit_trap:547                 :   [[ True \u003d\u003d \\T\\r\\u\\e ]]\n+ ./stack.sh:exit_trap:548                 :   echo \u0027exit_trap: cleaning up child processes\u0027\nexit_trap: cleaning up child processes\n```\n\noslo.messaging-tox-py310-func-scenario02:\n```\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(warn,messagingv2)\n--------------------------------------------------------------------------------------------------------------------\n\nCaptured traceback:\n~~~~~~~~~~~~~~~~~~~\n    Traceback (most recent call last):\n\n      File \"/home/zuul/src/opendev.org/openstack/oslo.messaging/oslo_messaging/tests/functional/notify/test_logger.py\", line 76, in test_logging\n    self.assertEqual(1, len(events))\n\n      File \"/home/zuul/src/opendev.org/openstack/oslo.messaging/.tox/py310-func-scenario02/lib/python3.10/site-packages/testtools/testcase.py\", line 419, in assertEqual\n    self.assertThat(observed, matcher, message)\n\n      File \"/home/zuul/src/opendev.org/openstack/oslo.messaging/.tox/py310-func-scenario02/lib/python3.10/site-packages/testtools/testcase.py\", line 509, in assertThat\n    raise mismatch_error\n\n    testtools.matchers._impl.MismatchError: 1 !\u003d 0\n```\n\n```\n------------------------------------------------------------------------------------------------------------------------  -----------\noslo_messaging.tests.functional.test_functional.CallTestCase.test_different_exchanges                                     23.249\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(error,messagingv2)     22.551\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(error,messaging)       22.550\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(warn,messaging)        22.531\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(critical,messaging)    22.531\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(info,messaging)        21.547\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(critical,messagingv2)  21.546\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(info,messagingv2)      21.545\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(debug,messagingv2)     21.544\noslo_messaging.tests.functional.notify.test_logger.LoggingNotificationHandlerTestCase.test_logging(warn,messagingv2)      21.540\nWARNING [pifpaf.util] `psutil.Popen(pid\u003d24091, status\u003d\u0027terminated\u0027, exitcode\u003d1, started\u003d\u002708:58:30\u0027)` is already gone, sending SIGKILL to its process group\nWARNING [pifpaf.util] `psutil.Popen(pid\u003d23400, status\u003d\u0027terminated\u0027, exitcode\u003d0, started\u003d\u002708:58:20\u0027)` is already gone, sending SIGKILL to its process group\nWARNING [pifpaf.util] `psutil.Popen(pid\u003d23351, status\u003d\u0027terminated\u0027, exitcode\u003d0, started\u003d\u002708:58:20\u0027)` is already gone, sending SIGKILL to its process group\nWARNING [pifpaf.util] `psutil.Popen(pid\u003d23256, status\u003d\u0027terminated\u0027, started\u003d\u002708:58:12\u0027)` is already gone, sending SIGKILL to its process group\n```","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":36083,"name":"Nikita Kushnarenko","display_name":"Nikita Kushnarenko","email":"NKushnarenko@itkey.com","username":"NickKush"},"change_message_id":"bdde5a713d0239a43b2dddc0166f27d28886bb15","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"590c47a6_c6a27387","updated":"2024-12-03 10:21:59.000000000","message":"recheck","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"change_message_id":"b2adfaa41959795e5d1a3de0fde6bc16c513c8f9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fb5f39d8_60f419fa","updated":"2024-12-09 08:26:32.000000000","message":"recheck","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"2e4eac7590d6bddce903cf1b6a42350e2b7de096","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"1a06823d_ba94dc75","updated":"2026-04-29 11:23:21.000000000","message":"recheck\ntest_resize_volume_backed_server_confirm failure is not related","commit_id":"8d12df2ea60caf60bd2896a2f67652587cbc4292"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"e469b0d8d4d8d898c31a0559325521362464b93d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":6,"id":"7dd13134_8059bf69","updated":"2026-05-19 17:43:28.000000000","message":"It\u0027s not clear to me if the change directly fixes the problem. Looking at the report, it sounds like internal retry mechanism does not work due to the whole consume timeout reaches BEFORE the ensure call gets the first error caused by node failure. I\u0027m wondering if we should rather look at how timeout are set in each layer and tune these correctly so that the overall consume timeout does reach first ?","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"34cc161fdde85b9774a463f9447ceb1a75545a55","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":6,"id":"13b1d6ce_782f3338","in_reply_to":"7dd13134_8059bf69","updated":"2026-05-22 14:59:15.000000000","message":"The aproach in this patch is to catch when connection is dead (not self.connection.connected), instead of raising a recoverable error (if I understand well that falls back to kombu autoretry, but that tries the same host) it tries trigger kombu\u0027s ensure_connection() to build the connection on a different host.\n\nAs I see tuning timeouts is not enough as the solution is to force the reconnect on a different host instead of the dead one.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"}],"oslo_messaging/_drivers/impl_rabbit.py":[{"author":{"_account_id":16137,"name":"Tobias Urdin","email":"tobias.urdin@binero.com","username":"tobasco"},"change_message_id":"b30011dee346b118897242965a1dd4b61345dd82","unresolved":true,"context_lines":[{"line_number":1457,"context_line":"                    self.connection.drain_events(timeout\u003dpoll_timeout)"},{"line_number":1458,"context_line":"                    return"},{"line_number":1459,"context_line":"                except socket.timeout:"},{"line_number":1460,"context_line":"                    if self._heartbeat_supported_and_enabled():"},{"line_number":1461,"context_line":"                        self._heartbeat_check()"},{"line_number":1462,"context_line":""},{"line_number":1463,"context_line":"                    poll_timeout \u003d timer.check_return("}],"source_content_type":"text/x-python","patch_set":2,"id":"0f6cafe9_62ee9206","line":1460,"updated":"2024-12-09 10:31:35.000000000","message":"Can you describe more about this change, why should we only check heartbeat when we get a timeout?\n\nA little bit up the code there is this comment which seems to indicate it should be run (twice every second) with that.\n\u003e      # NOTE(sileht): kombu recommend to run heartbeat_check every\n\u003e        # seconds, but we use a lock around the kombu connection\n\u003e        # so, to not lock to much this lock to most of the time do nothing\n\u003e        # expected waiting the events drain, we start heartbeat_check and\n\u003e        # retrieve the server heartbeat packet only two times more than\n\u003e        # the minimum required for the heartbeat works\n\u003e        # (heartbeat_timeout/heartbeat_rate/2.0, default kombu\n\u003e        # heartbeat_rate is 2)","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":36083,"name":"Nikita Kushnarenko","display_name":"Nikita Kushnarenko","email":"NKushnarenko@itkey.com","username":"NickKush"},"change_message_id":"b9c4cdd13ec1f4f2642e1d083f3ca96e7dd1058c","unresolved":true,"context_lines":[{"line_number":1457,"context_line":"                    self.connection.drain_events(timeout\u003dpoll_timeout)"},{"line_number":1458,"context_line":"                    return"},{"line_number":1459,"context_line":"                except socket.timeout:"},{"line_number":1460,"context_line":"                    if self._heartbeat_supported_and_enabled():"},{"line_number":1461,"context_line":"                        self._heartbeat_check()"},{"line_number":1462,"context_line":""},{"line_number":1463,"context_line":"                    poll_timeout \u003d timer.check_return("}],"source_content_type":"text/x-python","patch_set":2,"id":"82d0bac2_3d3b6656","line":1460,"in_reply_to":"0f6cafe9_62ee9206","updated":"2024-12-09 12:41:15.000000000","message":"This is a Kombu recommendation (undocumented as far as I know). \nAs an example, in their ConsumerMixin implementation, the heartbeat_check will only be called on socket.timeout, otherwise the heartbeat check will be called for every message received.\n\nThe drain_events would not produce any heartbeats, unless there are messages to consume and acknowledge. If the queue is idle/empty then the connection would raise socket.timeout, so it seemed appropriate to me to do a heartbeat at that point.\n\n\nThe comment above refers to _heartbeat_wait_timeout, which is mostly used in _heartbeat_thread_job.\n\nThe heartbeat thread is used when connection is a producer.\n```\n# NOTE(sileht): if purpose is PURPOSE_LISTEN\n# the consume code does the heartbeat stuff\n# we don\u0027t need a thread\n```\n\nThe consumer in this case will have a poll_timeout equal to the minimum value of `timeout` from AMQPListener.poll and _heartbeat_wait_timeout. So, we do not exceed the recommended heartbeat interval.\n\nI can make this change as a separate change so as not to mix two issues into one. \nLet me know how best to proceed!","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d40dba4de267331b19324d8ead9e4b51c84d3805","unresolved":true,"context_lines":[{"line_number":1457,"context_line":"                    self.connection.drain_events(timeout\u003dpoll_timeout)"},{"line_number":1458,"context_line":"                    return"},{"line_number":1459,"context_line":"                except socket.timeout:"},{"line_number":1460,"context_line":"                    if self._heartbeat_supported_and_enabled():"},{"line_number":1461,"context_line":"                        self._heartbeat_check()"},{"line_number":1462,"context_line":""},{"line_number":1463,"context_line":"                    poll_timeout \u003d timer.check_return("}],"source_content_type":"text/x-python","patch_set":2,"id":"e0402f67_25a48351","line":1460,"in_reply_to":"82d0bac2_3d3b6656","updated":"2026-04-28 12:17:05.000000000","message":"Thanks Nikita for the detailed explanation.\nI updated the comment after rebase, @tobias.urdin@binero.com if please check if we can keep it in one patch or cut into pieces to make it cleaner and consumable.","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":16137,"name":"Tobias Urdin","email":"tobias.urdin@binero.com","username":"tobasco"},"change_message_id":"9a1f5a353d835357c321b2a6be413163edd234af","unresolved":false,"context_lines":[{"line_number":1457,"context_line":"                    self.connection.drain_events(timeout\u003dpoll_timeout)"},{"line_number":1458,"context_line":"                    return"},{"line_number":1459,"context_line":"                except socket.timeout:"},{"line_number":1460,"context_line":"                    if self._heartbeat_supported_and_enabled():"},{"line_number":1461,"context_line":"                        self._heartbeat_check()"},{"line_number":1462,"context_line":""},{"line_number":1463,"context_line":"                    poll_timeout \u003d timer.check_return("}],"source_content_type":"text/x-python","patch_set":2,"id":"a61b46e4_7f6b0fe7","line":1460,"in_reply_to":"e0402f67_25a48351","updated":"2026-05-03 16:56:49.000000000","message":"LGTM","commit_id":"b434adad782963460852c24cb6a031f2b00127c1"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"e469b0d8d4d8d898c31a0559325521362464b93d","unresolved":true,"context_lines":[{"line_number":1099,"context_line":"            # NOTE(gsantomaggio): we must reraise this without"},{"line_number":1100,"context_line":"            # trigger error_callback"},{"line_number":1101,"context_line":"            raise"},{"line_number":1102,"context_line":"        except rpc_common.Timeout:"},{"line_number":1103,"context_line":"            # NOTE must reraise this without"},{"line_number":1104,"context_line":"            # trigger error_callback"},{"line_number":1105,"context_line":"            raise"}],"source_content_type":"text/x-python","patch_set":6,"id":"7d400df7_a9d7ad2e","line":1102,"range":{"start_line":1102,"start_character":15,"end_line":1102,"end_character":33},"updated":"2026-05-19 17:43:28.000000000","message":"I assume this is added here because error_callback eventually raises rpc_common.Timeout . Is that correct ?\n\nI saw a few additional logging implemented in erorr_callback and I\u0027m guessing that calling the callback is still useful.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"34cc161fdde85b9774a463f9447ceb1a75545a55","unresolved":true,"context_lines":[{"line_number":1099,"context_line":"            # NOTE(gsantomaggio): we must reraise this without"},{"line_number":1100,"context_line":"            # trigger error_callback"},{"line_number":1101,"context_line":"            raise"},{"line_number":1102,"context_line":"        except rpc_common.Timeout:"},{"line_number":1103,"context_line":"            # NOTE must reraise this without"},{"line_number":1104,"context_line":"            # trigger error_callback"},{"line_number":1105,"context_line":"            raise"}],"source_content_type":"text/x-python","patch_set":6,"id":"f585ae5e_8e0f199e","line":1102,"range":{"start_line":1102,"start_character":15,"end_line":1102,"end_character":33},"in_reply_to":"7d400df7_a9d7ad2e","updated":"2026-05-22 14:59:15.000000000","message":"yes, exactly, otherwise the error_callback call to be sure that the correct exception is handled up in the chain.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"cf0af716d322f46b745bf1b32ff455b0153a8295","unresolved":false,"context_lines":[{"line_number":1099,"context_line":"            # NOTE(gsantomaggio): we must reraise this without"},{"line_number":1100,"context_line":"            # trigger error_callback"},{"line_number":1101,"context_line":"            raise"},{"line_number":1102,"context_line":"        except rpc_common.Timeout:"},{"line_number":1103,"context_line":"            # NOTE must reraise this without"},{"line_number":1104,"context_line":"            # trigger error_callback"},{"line_number":1105,"context_line":"            raise"}],"source_content_type":"text/x-python","patch_set":6,"id":"a19d904f_5edce751","line":1102,"range":{"start_line":1102,"start_character":15,"end_line":1102,"end_character":33},"in_reply_to":"f585ae5e_8e0f199e","updated":"2026-06-04 15:32:17.000000000","message":"Done","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"e469b0d8d4d8d898c31a0559325521362464b93d","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"e95b986d_9f58e991","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"updated":"2026-05-19 17:43:28.000000000","message":"I\u0027m confused because kombu is supposed to trigger _ensure_connection in kombu layer which does the same thing. The only difference is that it may consume one retry cycle but I\u0027m unsure if that could fix the problem.\n\nhttps://github.com/celery/kombu/blob/817ebae4943e3481dc920e58681f5f5bff61fc1e/kombu/connection.py#L601","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"8685ed2094efec5feb92130d14fed855f5e486db","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"d5392314_0b393971","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"23c1a7c7_611764b2","updated":"2026-06-15 17:04:27.000000000","message":"First, I fully appreciate your great help here and sorry for my delayed response.\n\nAfter digging further into the logic and re-reading the original bug report, I\u0027m becoming confident that the core problem is caused by too short pool timeout set in consume calls\n https://github.com/openstack/oslo.messaging/blob/18.1.0/oslo_messaging/_drivers/amqpdriver.py#L435\n https://github.com/openstack/oslo.messaging/blob/18.1.0/oslo_messaging/_drivers/amqpdriver.py#L435\n \nLooking at kombu, connect_timeout is hard-coded to to 5 . So if _ensure_connection is called through the _ensured wrapper it first attempts to use the same host and then wait for 5 seconds. This makes the on_error callback to immediately fail because timeout is capped by ACK_REQUEUE_EVERY_SECONDS_MAX \u003d 5s ...\n https://github.com/celery/kombu/blob/v5.6.2/kombu/connection.py#L171\n\nThis behavior is what is indicated by https://bugs.launchpad.net/oslo.messaging/+bug/2077152/comments/1 .\n\nThe main reason this change \"fixes\" the problem is that it calls ensure_connection without that _recoverable_error_callback, which evaluates timeout.\n https://github.com/openstack/oslo.messaging/blob/18.1.0/oslo_messaging/_drivers/impl_rabbit.py#L1395-L1398\n\nThis is likely a bug which was surfaced by https://review.opendev.org/c/openstack/oslo.messaging/+/800564 .\n\nWhile I agree this is a potential solution, the change may also conflicts with the poll timeout set in upper layer (because ensure_connection is not interrupted after timeout expires).\n\nI\u0027ll give it a thought to see if I could come up with a better approach. One alternative solution I have in my mind is to bump ACK_REQUEUE_EVERY_SECONDS_MAX though I understand this has side-effect.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"cf0af716d322f46b745bf1b32ff455b0153a8295","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"b1c4743b_3d2c404a","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"abae829d_08ba479d","updated":"2026-06-04 15:32:17.000000000","message":"I\u0027m still trying to dig into this but I\u0027m wondering if you can point that \"autoretry\" you mentioned ? I was looking into Connection.autoretry in kombu but didn\u0027t see any relevant logic.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"57182b7466ebe83f744bab26e485ef4d8a12f080","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"23c1a7c7_611764b2","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"b1c4743b_3d2c404a","updated":"2026-06-05 11:38:52.000000000","message":"I sat over it again for hours also 😆 I try to write down how I understand combu\u0027s logic goes\n\n* OLD (raise recoverable_connection_errors[0]): _consume() runs inside kombu\u0027s ensure loop. When we raise recoverable_connection_errors[0], kombu\u0027s _ensured catches it\n(connection.py#L568-L585 (https://github.com/celery/kombu/blob/v5.6.2/kombu/connection.py#L568-L585)) and calls _ensure_connection. However, _ensure_connection first attempts to reconnect\nto the current host (the dead one) via _connection_factory (connection.py#L462 (https://github.com/celery/kombu/blob/v5.6.2/kombu/connection.py#L462)). Only after that attempt fails does on_error fire and call maybe_switch_next() to rotate to the next host (connection.py#L453 (https://github.com/celery/kombu/blob/v5.6.2/kombu/connection.py#L453)). \nEach failed attempt introduces backoff delays. \nIf the reconnection ultimately lands back on the same dead host (or oslo.messaging\u0027s channel state becomes inconsistent), _consume() sees not connected again and the cycle repeats.\n\n* NEW (self.ensure_connection()): We call _ensure_connection directly (via oslo.messaging\u0027s wrapper which also resets the channel). No exception is raised, \nso kombu\u0027s ensure loop has nothing to catch and no retry counter is consumed. ensure_connection starts its own retry_over_time loop with the failover_strategy (through on_error -\u003e maybe_switch_next -\u003e self.cycle()/self.failover_strategy()), connects to a live host, oslo.messaging resets its channel state, and _consume() proceeds normally.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"12599bf33c2fd71e526ddb75484849ff2b8e285f","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"5e2e4659_8551287f","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"cf6fae2e_e045d084","updated":"2026-06-16 11:48:40.000000000","message":"thanks Takashi for checking and your time","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"afe5f8cae8ec5eac9995fe7712ebcfda3b95c678","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"cf6fae2e_e045d084","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"d5392314_0b393971","updated":"2026-06-15 17:05:29.000000000","message":"Minor correction\n\n\u003e connect_timeout is hard-coded to to 5\n\nThis is not hard-coded. It just defaults to 5 but we don\u0027t override it so far.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"34cc161fdde85b9774a463f9447ceb1a75545a55","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"            # Kombu\u0027s ensure_connection utilises a failover strategy, instead"},{"line_number":1365,"context_line":"            # of retring to connect to the same host."},{"line_number":1366,"context_line":"            if not self.connection.connected:"},{"line_number":1367,"context_line":"                self.ensure_connection()"},{"line_number":1368,"context_line":""},{"line_number":1369,"context_line":"            while self._new_tags:"},{"line_number":1370,"context_line":"                for consumer, tag in self._consumers.items():"}],"source_content_type":"text/x-python","patch_set":6,"id":"abae829d_08ba479d","line":1367,"range":{"start_line":1367,"start_character":16,"end_line":1367,"end_character":40},"in_reply_to":"e95b986d_9f58e991","updated":"2026-05-22 14:59:15.000000000","message":"exactly, but kombu will trigger ensure_connection if the kombu ensure wrapper detects the error, but the original code called only recoverable_connection_errors and that is enough only for kombu to autoretry, which tries the same host (which is dead).\n\nThe old code raises a recoverable error that feeds into kombu\u0027s autoretry (same host), but calling ensure_connection we start kombu\u0027s failover_strategy. With this we can avoid a loop of retries to the same dead host, and retrying to a live node.\nSo we consume a retry cycle but the difference here is that after that the connection will be on a working node, not just retried on the dead one.","commit_id":"11e949b861794af4a31cd7829ebe6bc39b25391a"}]}