)]}'
{"id":"openstack%2Foslo.messaging~739175","triplet_id":"openstack%2Foslo.messaging~master~I2ae53340783e4044dab58035bc0992dc08145b53","project":"openstack/oslo.messaging","branch":"master","topic":"bug/1789177","hashtags":[],"change_id":"I2ae53340783e4044dab58035bc0992dc08145b53","subject":"Cancel consumer if queue down","status":"MERGED","created":"2020-07-03 07:55:55.000000000","updated":"2020-08-01 01:22:56.000000000","submitted":"2020-08-01 01:21:42.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":3,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"739175-1596244902844-7b2e085e","meta_rev_id":"6d3a309611662ee5bf1b38dbe190fabff38c6998","_number":739175,"virtual_id_number":739175,"owner":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2020-08-01 01:21:42.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"date":"2020-07-31 22:07:40.000000000","_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"all":[{"value":2,"date":"2020-07-31 17:30:16.000000000","_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":2,"date":"2020-07-31 08:28:57.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"all":[{"value":1,"date":"2020-07-31 17:29:59.000000000","_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Backport-Candidate":{"all":[{"value":0,"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"}],"values":{"-2":"Do Not Backport","-1":"Not A Backport Candidate"," 0":"Backport Review Needed","+1":"Proposed Backport","+2":"Should Backport"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-07-31 08:28:57.000000000","updated_by":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"reviewer":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"state":"REVIEWER"},{"updated":"2020-07-31 17:29:59.000000000","updated_by":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"reviewer":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"state":"REVIEWER"},{"updated":"2020-08-01 01:21:42.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"44ae42d2ba1b2364779398467ce1871dd41c6274","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-03 07:55:55.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"2a23bf55e8bc8be8bc45240ec4e9cd8850fbe7fb","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-03 08:04:01.000000000","message":"Uploaded patch set 2: Commit message was updated.","accounts_in_message":[],"_revision_number":2},{"id":"4e8e7f87830f68189e2fddb710fcc778b4b8741c","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-03 08:44:43.000000000","message":"Patch Set 2: Backport-Candidate+1\n\nHello,\n\nInteresting changes, +1 for the rationale of these changes, waiting for CI and a feedback from a more experienced person than me on rabbitmq\u0027s topics (Ken by example).\n\nAFAIK here\u0027s roughly how it happens:\n\nNormally the \"no exchange\" errors are some variation of that scenario, and are \"normal\" at the application layer.  Oslo.messaging is designed to retry against the exchange for some time in hopes that it reappears.  If not, it should discard the reply and stop logging that error.\n\nExample:\n\nService A creates an exchange/queue pair reply_XYZ for receiving RPC responses.  These are created with auto_delete\u003dtrue.\n\n- A makes an RPC call to service B.\n\n- B receives the RPC request from A, and does some work.\n\n- A stops or is disconnected from RabbitMQ for $reasons.\n\n- RabbitMQ notices there are zero consumers on reply_XYZ, and auto deletes the queue.  Because the exchange is not bound to any queues now, it is also auto deleted.\n\n- B is done working, and attempts to publish to exchange reply_XYZ, but it is gone.\n\n- B will retry for some time (Offhand, I believe once per second for 60 seconds), hoping that A reconnects and re-declares the queue/exchange.  During this time, you will get this error repeatedly.\n\n- Eventually either (a) A comes back, re-declares reply_XYZ, and B is able to send the reply, or (b) service B gives up after the timeout and stops trying to send the reply.\n\n\nSo it\u0027s not necessarily indicative of a problem.  It\u0027s supposed to work that way in the general case.  However, it *could* be indicative of a problem, probably related to a partition, either ongoing or at some point in the past.  There\u0027s not really any rule to say for sure, you have to examine the context on a case-by-case basis.\n\nI think these changes could help us to implement an auto-healing after rabbitmq crash.\n\nI wondering if we could introduce some related functional tests to simulate similar scenario and ensure how it will behave in this situation.\n\nThoughts?\n\nSide note: this LP could be related to this topic too \u003d\u003e https://bugs.launchpad.net/mos/+bug/1609741","accounts_in_message":[],"_revision_number":2},{"id":"eea5fd08136e370ee26143ea19ef58f3423b752b","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-03 09:11:37.000000000","message":"Patch Set 2: -Backport-Candidate Code-Review+1\n\nBy waiting more feedbacks please can you add a release note to highlight these changes.","accounts_in_message":[],"_revision_number":2},{"id":"894006575bc168651179191ab46e01ce7beb1554","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-03 09:17:19.000000000","message":"Patch Set 2:\n\n\u003e Hello,\n \u003e \n \u003e Interesting changes, +1 for the rationale of these changes, waiting\n \u003e for CI and a feedback from a more experienced person than me on\n \u003e rabbitmq\u0027s topics (Ken by example).\n \u003e \n \u003e AFAIK here\u0027s roughly how it happens:\n \u003e \n \u003e Normally the \"no exchange\" errors are some variation of that\n \u003e scenario, and are \"normal\" at the application layer. \n \u003e Oslo.messaging is designed to retry against the exchange for some\n \u003e time in hopes that it reappears.  If not, it should discard the\n \u003e reply and stop logging that error.\n \u003e \n \u003e Example:\n \u003e \n \u003e Service A creates an exchange/queue pair reply_XYZ for receiving\n \u003e RPC responses.  These are created with auto_delete\u003dtrue.\n \u003e \n \u003e - A makes an RPC call to service B.\n \u003e \n \u003e - B receives the RPC request from A, and does some work.\n \u003e \n \u003e - A stops or is disconnected from RabbitMQ for $reasons.\n \u003e \n \u003e - RabbitMQ notices there are zero consumers on reply_XYZ, and auto\n \u003e deletes the queue.  Because the exchange is not bound to any queues\n \u003e now, it is also auto deleted.\n \u003e \n \u003e - B is done working, and attempts to publish to exchange reply_XYZ,\n \u003e but it is gone.\n \u003e \n \u003e - B will retry for some time (Offhand, I believe once per second\n \u003e for 60 seconds), hoping that A reconnects and re-declares the\n \u003e queue/exchange.  During this time, you will get this error\n \u003e repeatedly.\n \u003e \n \u003e - Eventually either (a) A comes back, re-declares reply_XYZ, and B\n \u003e is able to send the reply, or (b) service B gives up after the\n \u003e timeout and stops trying to send the reply.\n \u003e \n \u003e \n \u003e So it\u0027s not necessarily indicative of a problem.  It\u0027s supposed to\n \u003e work that way in the general case.  However, it *could* be\n \u003e indicative of a problem, probably related to a partition, either\n \u003e ongoing or at some point in the past.  There\u0027s not really any rule\n \u003e to say for sure, you have to examine the context on a case-by-case\n \u003e basis.\n \u003e \n \u003e I think these changes could help us to implement an auto-healing\n \u003e after rabbitmq crash.\n \u003e \n \u003e I wondering if we could introduce some related functional tests to\n \u003e simulate similar scenario and ensure how it will behave in this\n \u003e situation.\n \u003e \n \u003e Thoughts?\n \u003e \n \u003e Side note: this LP could be related to this topic too \u003d\u003e\n \u003e https://bugs.launchpad.net/mos/+bug/1609741\n\nHi, thank you for reply. btw the period is 0.25 imo https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqpdriver.py#L158...\n\nthis patch is intended to eventually make the agent aware of reply_* queue is gone, and consumer should be re-declared. actually it relies on the rabbitmq cluster returns to normal.\n\npreviously I saw agents not redeclare queues because both channels and connections are intact, seems that it is possible because sometimes rabbitmq server will not notify agent queue is gone.\n\nAs for test, I agree it is necessary but I am struggling to think of a way to design one, do you have any idea in mind? thank you.","accounts_in_message":[],"_revision_number":2},{"id":"dec0af6413aebbdf214e4d8fbf5eea4fd5df1228","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-03 09:18:46.000000000","message":"Patch Set 2:\n\n\u003e By waiting more feedbacks please can you add a release note to\n \u003e highlight these changes.\n\nsure.","accounts_in_message":[],"_revision_number":2},{"id":"a1a7bb1f7dbe9071ab840234a2a6f8a51088295e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-03 09:58:22.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/72e922b040be48429cf2b210e8e08873 : SUCCESS in 1h 38m 42s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/0887e205acd84eec8a0ef5b1ca51da01 : SUCCESS in 8m 13s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/8d1cab50651c46babed185a99bd2a22c : SUCCESS in 8m 45s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/7b8fdd66e66b4ce3a619a917bb17d6e5 : SUCCESS in 6m 37s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/75b55270bb80475d81c968069009f5be : SUCCESS in 6m 55s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/4d12789f98ef4ac9924641a56eeacdf9 : SUCCESS in 7m 47s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/bce3c6f0216f456890ecfaf9c72dafd1 : SUCCESS in 6m 40s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/c31a7f2e6ef44ad388c2c2dd5544cad4 : SUCCESS in 11m 44s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/77e14089e25a4712a6344adc31059567 : SUCCESS in 8m 05s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/3f571de924ce4192a76548897b8fc625 : SUCCESS in 7m 24s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/7b048dde1d6b4c73915f248b9327b412 : SUCCESS in 6m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/4750878d10b348d9b5b4197ef8845708 : SUCCESS in 1h 31m 41s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/bff173af69c54597ae1baf988e4baea7 : FAILURE in 1h 00m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/8160dec23502452bbc35659dc52eb5b4 : SUCCESS in 1h 47m 38s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://zuul.opendev.org/t/openstack/build/27eb433a8c6841579e24d903481179e1 : SUCCESS in 1h 03m 32s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://zuul.opendev.org/t/openstack/build/e8d9b2f7c60b4d40a19ae0b9f8637ef3 : FAILURE in 9m 59s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"c215b19fc69f2ab00b7a545a6df738e3fc5b1fd2","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-03 12:03:29.000000000","message":"Patch Set 2:\n\n\u003e \u003e Hello,\n \u003e \u003e\n \u003e \u003e Interesting changes, +1 for the rationale of these changes,\n \u003e waiting\n \u003e \u003e for CI and a feedback from a more experienced person than me on\n \u003e \u003e rabbitmq\u0027s topics (Ken by example).\n \u003e \u003e\n \u003e \u003e AFAIK here\u0027s roughly how it happens:\n \u003e \u003e\n \u003e \u003e Normally the \"no exchange\" errors are some variation of that\n \u003e \u003e scenario, and are \"normal\" at the application layer.\n \u003e \u003e Oslo.messaging is designed to retry against the exchange for some\n \u003e \u003e time in hopes that it reappears.  If not, it should discard the\n \u003e \u003e reply and stop logging that error.\n \u003e \u003e\n \u003e \u003e Example:\n \u003e \u003e\n \u003e \u003e Service A creates an exchange/queue pair reply_XYZ for receiving\n \u003e \u003e RPC responses.  These are created with auto_delete\u003dtrue.\n \u003e \u003e\n \u003e \u003e - A makes an RPC call to service B.\n \u003e \u003e\n \u003e \u003e - B receives the RPC request from A, and does some work.\n \u003e \u003e\n \u003e \u003e - A stops or is disconnected from RabbitMQ for $reasons.\n \u003e \u003e\n \u003e \u003e - RabbitMQ notices there are zero consumers on reply_XYZ, and\n \u003e auto\n \u003e \u003e deletes the queue.  Because the exchange is not bound to any\n \u003e queues\n \u003e \u003e now, it is also auto deleted.\n \u003e \u003e\n \u003e \u003e - B is done working, and attempts to publish to exchange\n \u003e reply_XYZ,\n \u003e \u003e but it is gone.\n \u003e \u003e\n \u003e \u003e - B will retry for some time (Offhand, I believe once per second\n \u003e \u003e for 60 seconds), hoping that A reconnects and re-declares the\n \u003e \u003e queue/exchange.  During this time, you will get this error\n \u003e \u003e repeatedly.\n \u003e \u003e\n \u003e \u003e - Eventually either (a) A comes back, re-declares reply_XYZ, and\n \u003e B\n \u003e \u003e is able to send the reply, or (b) service B gives up after the\n \u003e \u003e timeout and stops trying to send the reply.\n \u003e \u003e\n \u003e \u003e\n \u003e \u003e So it\u0027s not necessarily indicative of a problem.  It\u0027s supposed\n \u003e to\n \u003e \u003e work that way in the general case.  However, it *could* be\n \u003e \u003e indicative of a problem, probably related to a partition, either\n \u003e \u003e ongoing or at some point in the past.  There\u0027s not really any\n \u003e rule\n \u003e \u003e to say for sure, you have to examine the context on a\n \u003e case-by-case\n \u003e \u003e basis.\n \u003e \u003e\n \u003e \u003e I think these changes could help us to implement an auto-healing\n \u003e \u003e after rabbitmq crash.\n \u003e \u003e\n \u003e \u003e I wondering if we could introduce some related functional tests\n \u003e to\n \u003e \u003e simulate similar scenario and ensure how it will behave in this\n \u003e \u003e situation.\n \u003e \u003e\n \u003e \u003e Thoughts?\n \u003e \u003e\n \u003e \u003e Side note: this LP could be related to this topic too \u003d\u003e\n \u003e \u003e https://bugs.launchpad.net/mos/+bug/1609741\n \u003e \n \u003e Hi, thank you for reply. btw the period is 0.25 imo\n \u003e https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqpdriver.py#L158...\n \u003e \n \u003e this patch is intended to eventually make the agent aware of\n \u003e reply_* queue is gone, and consumer should be re-declared. actually\n \u003e it relies on the rabbitmq cluster returns to normal.\n \u003e \n \u003e previously I saw agents not redeclare queues because both channels\n \u003e and connections are intact, seems that it is possible because\n \u003e sometimes rabbitmq server will not notify agent queue is gone.\n \u003e \n \u003e As for test, I agree it is necessary but I am struggling to think\n \u003e of a way to design one, do you have any idea in mind? thank you.\n\nHmmm I need to dive deep in our tests to make an idea, I\u0027ll take a look ASAP and I\u0027ll try to bring back with some track to follow.","accounts_in_message":[],"_revision_number":2},{"id":"123e70f8ae1c360936978b13132e9125a7a97f5c","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-06 11:17:03.000000000","message":"Patch Set 2:\n\nHello,\n\nHere are some news ideas, possible track to follow to implement functinal tests.\n\n \u003e \u003e Hello,\n \u003e \u003e\n \u003e \u003e Interesting changes, +1 for the rationale of these changes,\n \u003e waiting\n \u003e \u003e for CI and a feedback from a more experienced person than me on\n \u003e \u003e rabbitmq\u0027s topics (Ken by example).\n \u003e \u003e\n \u003e \u003e AFAIK here\u0027s roughly how it happens:\n \u003e \u003e\n \u003e \u003e Normally the \"no exchange\" errors are some variation of that\n \u003e \u003e scenario, and are \"normal\" at the application layer.\n \u003e \u003e Oslo.messaging is designed to retry against the exchange for some\n \u003e \u003e time in hopes that it reappears.  If not, it should discard the\n \u003e \u003e reply and stop logging that error.\n \u003e \u003e\n \u003e \u003e Example:\n \u003e \u003e\n \u003e \u003e Service A creates an exchange/queue pair reply_XYZ for receiving\n \u003e \u003e RPC responses.  These are created with auto_delete\u003dtrue.\n \u003e \u003e\n \u003e \u003e - A makes an RPC call to service B.\n \u003e \u003e\n \u003e \u003e - B receives the RPC request from A, and does some work.\n \u003e \u003e\n \u003e \u003e - A stops or is disconnected from RabbitMQ for $reasons.\n \u003e \u003e\n \u003e \u003e - RabbitMQ notices there are zero consumers on reply_XYZ, and\n \u003e auto\n \u003e \u003e deletes the queue.  Because the exchange is not bound to any\n \u003e queues\n \u003e \u003e now, it is also auto deleted.\n \u003e \u003e\n \u003e \u003e - B is done working, and attempts to publish to exchange\n \u003e reply_XYZ,\n \u003e \u003e but it is gone.\n \u003e \u003e\n \u003e \u003e - B will retry for some time (Offhand, I believe once per second\n \u003e \u003e for 60 seconds), hoping that A reconnects and re-declares the\n \u003e \u003e queue/exchange.  During this time, you will get this error\n \u003e \u003e repeatedly.\n \u003e \u003e\n \u003e \u003e - Eventually either (a) A comes back, re-declares reply_XYZ, and\n \u003e B\n \u003e \u003e is able to send the reply, or (b) service B gives up after the\n \u003e \u003e timeout and stops trying to send the reply.\n \u003e \u003e\n \u003e \u003e\n \u003e \u003e So it\u0027s not necessarily indicative of a problem.  It\u0027s supposed\n \u003e to\n \u003e \u003e work that way in the general case.  However, it *could* be\n \u003e \u003e indicative of a problem, probably related to a partition, either\n \u003e \u003e ongoing or at some point in the past.  There\u0027s not really any\n \u003e rule\n \u003e \u003e to say for sure, you have to examine the context on a\n \u003e case-by-case\n \u003e \u003e basis.\n \u003e \u003e\n \u003e \u003e I think these changes could help us to implement an auto-healing\n \u003e \u003e after rabbitmq crash.\n \u003e \u003e\n \u003e \u003e I wondering if we could introduce some related functional tests\n \u003e to\n \u003e \u003e simulate similar scenario and ensure how it will behave in this\n \u003e \u003e situation.\n \u003e \u003e\n \u003e \u003e Thoughts?\n \u003e \u003e\n \u003e \u003e Side note: this LP could be related to this topic too \u003d\u003e\n \u003e \u003e https://bugs.launchpad.net/mos/+bug/1609741\n \u003e \n \u003e Hi, thank you for reply. btw the period is 0.25 imo\n \u003e https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqpdriver.py#L158...\n \u003e \n \u003e this patch is intended to eventually make the agent aware of\n \u003e reply_* queue is gone, and consumer should be re-declared. actually\n \u003e it relies on the rabbitmq cluster returns to normal.\n \u003e \n \u003e previously I saw agents not redeclare queues because both channels\n \u003e and connections are intact, seems that it is possible because\n \u003e sometimes rabbitmq server will not notify agent queue is gone.\n \u003e \n \u003e As for test, I agree it is necessary but I am struggling to think\n \u003e of a way to design one, do you have any idea in mind? thank you.\n\nI think we could inspire from the rabbitmq failovers tests [1] to implement something related to these changes and our use case.\n\npifpaf allow us to handle rabbit nodes and especially stopping them during execution, so maybe we could try something in this way by creating a consumer and a publisher who will communicate over a reply queue and through mocking some kombu methods (delete [2], unbind_from [3]) check if these are called to ensure us that these changes works as expected. Note that I\u0027m not sure about which exact kombu\u0027s methods we should to mock, this is just suppositions about who of them could be called in this situation.\n\nBy example I think we could mock.assert_called for kombu\u0027s method previously quoted.\n\nThoughts?\n\n[1] https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/tests/functional/test_rabbitmq.py#L41,L103\n[2] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L749\n[3] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L771\n[4] https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.assert_called","accounts_in_message":[],"_revision_number":2},{"id":"40a11cc4aea0fadcfc450cd66d60bd81bdf65d13","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-06 12:48:38.000000000","message":"Patch Set 2:\n\n\u003e Hello,\n \u003e \n \u003e Here are some news ideas, possible track to follow to implement\n \u003e functinal tests.\n \u003e \n \u003e \u003e \u003e Hello,\n \u003e \u003e \u003e\n \u003e \u003e \u003e Interesting changes, +1 for the rationale of these changes,\n \u003e \u003e waiting\n \u003e \u003e \u003e for CI and a feedback from a more experienced person than me on\n \u003e \u003e \u003e rabbitmq\u0027s topics (Ken by example).\n \u003e \u003e \u003e\n \u003e \u003e \u003e AFAIK here\u0027s roughly how it happens:\n \u003e \u003e \u003e\n \u003e \u003e \u003e Normally the \"no exchange\" errors are some variation of that\n \u003e \u003e \u003e scenario, and are \"normal\" at the application layer.\n \u003e \u003e \u003e Oslo.messaging is designed to retry against the exchange for\n \u003e some\n \u003e \u003e \u003e time in hopes that it reappears.  If not, it should discard the\n \u003e \u003e \u003e reply and stop logging that error.\n \u003e \u003e \u003e\n \u003e \u003e \u003e Example:\n \u003e \u003e \u003e\n \u003e \u003e \u003e Service A creates an exchange/queue pair reply_XYZ for\n \u003e receiving\n \u003e \u003e \u003e RPC responses.  These are created with auto_delete\u003dtrue.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - A makes an RPC call to service B.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - B receives the RPC request from A, and does some work.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - A stops or is disconnected from RabbitMQ for $reasons.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - RabbitMQ notices there are zero consumers on reply_XYZ, and\n \u003e \u003e auto\n \u003e \u003e \u003e deletes the queue.  Because the exchange is not bound to any\n \u003e \u003e queues\n \u003e \u003e \u003e now, it is also auto deleted.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - B is done working, and attempts to publish to exchange\n \u003e \u003e reply_XYZ,\n \u003e \u003e \u003e but it is gone.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - B will retry for some time (Offhand, I believe once per\n \u003e second\n \u003e \u003e \u003e for 60 seconds), hoping that A reconnects and re-declares the\n \u003e \u003e \u003e queue/exchange.  During this time, you will get this error\n \u003e \u003e \u003e repeatedly.\n \u003e \u003e \u003e\n \u003e \u003e \u003e - Eventually either (a) A comes back, re-declares reply_XYZ,\n \u003e and\n \u003e \u003e B\n \u003e \u003e \u003e is able to send the reply, or (b) service B gives up after the\n \u003e \u003e \u003e timeout and stops trying to send the reply.\n \u003e \u003e \u003e\n \u003e \u003e \u003e\n \u003e \u003e \u003e So it\u0027s not necessarily indicative of a problem.  It\u0027s supposed\n \u003e \u003e to\n \u003e \u003e \u003e work that way in the general case.  However, it *could* be\n \u003e \u003e \u003e indicative of a problem, probably related to a partition,\n \u003e either\n \u003e \u003e \u003e ongoing or at some point in the past.  There\u0027s not really any\n \u003e \u003e rule\n \u003e \u003e \u003e to say for sure, you have to examine the context on a\n \u003e \u003e case-by-case\n \u003e \u003e \u003e basis.\n \u003e \u003e \u003e\n \u003e \u003e \u003e I think these changes could help us to implement an\n \u003e auto-healing\n \u003e \u003e \u003e after rabbitmq crash.\n \u003e \u003e \u003e\n \u003e \u003e \u003e I wondering if we could introduce some related functional tests\n \u003e \u003e to\n \u003e \u003e \u003e simulate similar scenario and ensure how it will behave in this\n \u003e \u003e \u003e situation.\n \u003e \u003e \u003e\n \u003e \u003e \u003e Thoughts?\n \u003e \u003e \u003e\n \u003e \u003e \u003e Side note: this LP could be related to this topic too \u003d\u003e\n \u003e \u003e \u003e https://bugs.launchpad.net/mos/+bug/1609741\n \u003e \u003e\n \u003e \u003e Hi, thank you for reply. btw the period is 0.25 imo\n \u003e \u003e https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqpdriver.py#L158...\n \u003e \u003e\n \u003e \u003e this patch is intended to eventually make the agent aware of\n \u003e \u003e reply_* queue is gone, and consumer should be re-declared.\n \u003e actually\n \u003e \u003e it relies on the rabbitmq cluster returns to normal.\n \u003e \u003e\n \u003e \u003e previously I saw agents not redeclare queues because both\n \u003e channels\n \u003e \u003e and connections are intact, seems that it is possible because\n \u003e \u003e sometimes rabbitmq server will not notify agent queue is gone.\n \u003e \u003e\n \u003e \u003e As for test, I agree it is necessary but I am struggling to think\n \u003e \u003e of a way to design one, do you have any idea in mind? thank you.\n \u003e \n \u003e I think we could inspire from the rabbitmq failovers tests [1] to\n \u003e implement something related to these changes and our use case.\n \u003e \n \u003e pifpaf allow us to handle rabbit nodes and especially stopping them\n \u003e during execution, so maybe we could try something in this way by\n \u003e creating a consumer and a publisher who will communicate over a\n \u003e reply queue and through mocking some kombu methods (delete [2],\n \u003e unbind_from [3]) check if these are called to ensure us that these\n \u003e changes works as expected. Note that I\u0027m not sure about which exact\n \u003e kombu\u0027s methods we should to mock, this is just suppositions about\n \u003e who of them could be called in this situation.\n \u003e \n \u003e By example I think we could mock.assert_called for kombu\u0027s method\n \u003e previously quoted.\n \u003e \n \u003e Thoughts?\n \u003e \n \u003e [1] https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/tests/functional/test_rabbitmq.py#L41,L103\n \u003e [2] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L749\n \u003e [3] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L771\n \u003e [4] https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.assert_called\n\nhmm.. actually this change will not call \u0027delete\u0027 or \u0027unbind\u0027 on reply queue. The \u0027unbind\u0027 process is notified from rabbitmq server, and consumer which receives the message will need to bind again..","accounts_in_message":[],"_revision_number":2},{"id":"6fdd2037b4bce67cc4666ba8d327bdb6da62d7e5","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-08 10:00:04.000000000","message":"Patch Set 2:\n\n\u003e \u003e Hello,\n \u003e \u003e\n \u003e \u003e Here are some news ideas, possible track to follow to implement\n \u003e \u003e functinal tests.\n \u003e \u003e\n \u003e \u003e \u003e \u003e Hello,\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Interesting changes, +1 for the rationale of these changes,\n \u003e \u003e \u003e waiting\n \u003e \u003e \u003e \u003e for CI and a feedback from a more experienced person than me\n \u003e on\n \u003e \u003e \u003e \u003e rabbitmq\u0027s topics (Ken by example).\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e AFAIK here\u0027s roughly how it happens:\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Normally the \"no exchange\" errors are some variation of that\n \u003e \u003e \u003e \u003e scenario, and are \"normal\" at the application layer.\n \u003e \u003e \u003e \u003e Oslo.messaging is designed to retry against the exchange for\n \u003e \u003e some\n \u003e \u003e \u003e \u003e time in hopes that it reappears.  If not, it should discard\n \u003e the\n \u003e \u003e \u003e \u003e reply and stop logging that error.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Example:\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Service A creates an exchange/queue pair reply_XYZ for\n \u003e \u003e receiving\n \u003e \u003e \u003e \u003e RPC responses.  These are created with auto_delete\u003dtrue.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - A makes an RPC call to service B.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - B receives the RPC request from A, and does some work.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - A stops or is disconnected from RabbitMQ for $reasons.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - RabbitMQ notices there are zero consumers on reply_XYZ, and\n \u003e \u003e \u003e auto\n \u003e \u003e \u003e \u003e deletes the queue.  Because the exchange is not bound to any\n \u003e \u003e \u003e queues\n \u003e \u003e \u003e \u003e now, it is also auto deleted.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - B is done working, and attempts to publish to exchange\n \u003e \u003e \u003e reply_XYZ,\n \u003e \u003e \u003e \u003e but it is gone.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - B will retry for some time (Offhand, I believe once per\n \u003e \u003e second\n \u003e \u003e \u003e \u003e for 60 seconds), hoping that A reconnects and re-declares the\n \u003e \u003e \u003e \u003e queue/exchange.  During this time, you will get this error\n \u003e \u003e \u003e \u003e repeatedly.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e - Eventually either (a) A comes back, re-declares reply_XYZ,\n \u003e \u003e and\n \u003e \u003e \u003e B\n \u003e \u003e \u003e \u003e is able to send the reply, or (b) service B gives up after\n \u003e the\n \u003e \u003e \u003e \u003e timeout and stops trying to send the reply.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e So it\u0027s not necessarily indicative of a problem.  It\u0027s\n \u003e supposed\n \u003e \u003e \u003e to\n \u003e \u003e \u003e \u003e work that way in the general case.  However, it *could* be\n \u003e \u003e \u003e \u003e indicative of a problem, probably related to a partition,\n \u003e \u003e either\n \u003e \u003e \u003e \u003e ongoing or at some point in the past.  There\u0027s not really any\n \u003e \u003e \u003e rule\n \u003e \u003e \u003e \u003e to say for sure, you have to examine the context on a\n \u003e \u003e \u003e case-by-case\n \u003e \u003e \u003e \u003e basis.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e I think these changes could help us to implement an\n \u003e \u003e auto-healing\n \u003e \u003e \u003e \u003e after rabbitmq crash.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e I wondering if we could introduce some related functional\n \u003e tests\n \u003e \u003e \u003e to\n \u003e \u003e \u003e \u003e simulate similar scenario and ensure how it will behave in\n \u003e this\n \u003e \u003e \u003e \u003e situation.\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Thoughts?\n \u003e \u003e \u003e \u003e\n \u003e \u003e \u003e \u003e Side note: this LP could be related to this topic too \u003d\u003e\n \u003e \u003e \u003e \u003e https://bugs.launchpad.net/mos/+bug/1609741\n \u003e \u003e \u003e\n \u003e \u003e \u003e Hi, thank you for reply. btw the period is 0.25 imo\n \u003e \u003e \u003e https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqpdriver.py#L158...\n \u003e \u003e \u003e\n \u003e \u003e \u003e this patch is intended to eventually make the agent aware of\n \u003e \u003e \u003e reply_* queue is gone, and consumer should be re-declared.\n \u003e \u003e actually\n \u003e \u003e \u003e it relies on the rabbitmq cluster returns to normal.\n \u003e \u003e \u003e\n \u003e \u003e \u003e previously I saw agents not redeclare queues because both\n \u003e \u003e channels\n \u003e \u003e \u003e and connections are intact, seems that it is possible because\n \u003e \u003e \u003e sometimes rabbitmq server will not notify agent queue is gone.\n \u003e \u003e \u003e\n \u003e \u003e \u003e As for test, I agree it is necessary but I am struggling to\n \u003e think\n \u003e \u003e \u003e of a way to design one, do you have any idea in mind? thank\n \u003e you.\n \u003e \u003e\n \u003e \u003e I think we could inspire from the rabbitmq failovers tests [1] to\n \u003e \u003e implement something related to these changes and our use case.\n \u003e \u003e\n \u003e \u003e pifpaf allow us to handle rabbit nodes and especially stopping\n \u003e them\n \u003e \u003e during execution, so maybe we could try something in this way by\n \u003e \u003e creating a consumer and a publisher who will communicate over a\n \u003e \u003e reply queue and through mocking some kombu methods (delete [2],\n \u003e \u003e unbind_from [3]) check if these are called to ensure us that\n \u003e these\n \u003e \u003e changes works as expected. Note that I\u0027m not sure about which\n \u003e exact\n \u003e \u003e kombu\u0027s methods we should to mock, this is just suppositions\n \u003e about\n \u003e \u003e who of them could be called in this situation.\n \u003e \u003e\n \u003e \u003e By example I think we could mock.assert_called for kombu\u0027s method\n \u003e \u003e previously quoted.\n \u003e \u003e\n \u003e \u003e Thoughts?\n \u003e \u003e\n \u003e \u003e [1] https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/tests/functional/test_rabbitmq.py#L41,L103\n \u003e \u003e [2] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L749\n \u003e \u003e [3] https://github.com/celery/kombu/blob/93242848b8eda6532f2928e7058fff5fe3023115/kombu/entity.py#L771\n \u003e \u003e [4] https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.assert_called\n \u003e \n \u003e hmm.. actually this change will not call \u0027delete\u0027 or \u0027unbind\u0027 on\n \u003e reply queue. The \u0027unbind\u0027 process is notified from rabbitmq server,\n \u003e and consumer which receives the message will need to bind again..\n\nGood point, I\u0027ve no idea how to simulate a rabbitmq split brain with our available tools (pifpaf, CI etc...), so if functional tests are too complicated to implement we could continue without them.\n\nAnyway I don\u0027t think these changes can hurt.\n\nJust one last thing, please add a release note to highlight how oslo.messaging will behave with these changes.","accounts_in_message":[],"_revision_number":2},{"id":"7b71ba21fd2d111893fa79844b0a7e00a19f3696","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-10 16:03:43.000000000","message":"Patch Set 2: Code-Review+1\n\n+2 pending on addition of release note.\n\nThis patch looks like a good addition.  Unfortunately as you all point out there\u0027s really no existing test infrastructure we can use to ensure this patch doesn\u0027t bit rot.\n\nI\u0027d recommend waiting a bit on backporting this to stable branches until after it has been released.  Give folks some time to \"kick the tires\" first.","accounts_in_message":[],"_revision_number":2},{"id":"c070788a02fce1489d0048375165df7546d90de7","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-16 16:45:44.000000000","message":"Patch Set 2: Code-Review-1\n\nI\u0027ve spent some time trying to understand the semantics of the \"x-cancel-on-ha-failover\" consumer argument.  I\u0027m a little concerned about having this turned on by default.  From the rabbitmq docs [1] it appears that when failover to the mirrored queue occurs there can be redelivery of messages that are in doubt.  See [1], but I\u0027ve included a bit from below:\n\n\"When a mirrored queue fails over, knowledge of which messages have been sent to which consumer is lost, and therefore all unacknowledged messages are redelivered with the redelivered flag set. Consumers may wish to know this is going to happen.\"\n\n\"If so, they can consume with the argument x-cancel-on-ha-failover set to true. Their consuming will then be cancelled on failover and a consumer cancellation notification sent. It is then the consumer’s responsibility to reissue basic.consume to start consuming again.\"\n\nThis re-delivery aspect concerns me, especially in the case of an RPC Server where redelivered RPC requests may be executed twice.\n\nSo I\u0027m thinking that there\u0027s some risk to this patch, especially since we cannot reliably test it.\n\nThat said, I\u0027m not opposed to refactoring this patch to make the use of x-cancel-on-ha-behavior a configurable option (default is off).\n\n\n[1] https://rabbitmq.docs.pivotal.io/37/rabbit-web-docs/ha.html","accounts_in_message":[],"_revision_number":2},{"id":"89291b3555b7f7852135b1458f243f5d8900368a","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-19 00:20:12.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"e25a1dc144afcaffbf86a8f8a5852b23469cee6e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-19 01:45:08.000000000","message":"Patch Set 3: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/b26ca0bea450407fb6b21f3a0ccb4fbc : SUCCESS in 1h 17m 01s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/faaa788d04364770aa0a1f6d6e911b96 : SUCCESS in 5m 30s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/765454ccaa47483ab3d063b08f054080 : SUCCESS in 5m 30s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/63d79c1738884dd09895cc9a3de9de00 : SUCCESS in 5m 27s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/af7b8dbcc53845dab9c85f27d779465f : SUCCESS in 5m 34s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/a8fc44795f7546e2acd7202295a82832 : SUCCESS in 6m 14s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/038652eed2c2435ba0a08f5b6446de3c : SUCCESS in 4m 34s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/d42c868202c34cceaa9d0c836cd2535a : SUCCESS in 2m 59s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/81e66bbcbc74476192adee17dbb6039a : TIMED_OUT in 41m 03s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/1a2410dcbab8445c936ec28cdccaa177 : TIMED_OUT in 40m 47s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/16c23df616f84791a60d74ce32fc1270 : SUCCESS in 6m 23s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/d5afe9d2715a43f5968ed6d967d2bf18 : SUCCESS in 5m 36s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/542c429d287f4c1b950bd8b420dd947a : SUCCESS in 1h 22m 24s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/c3cf607098dd4389a24a2518d9ef2f2f : SUCCESS in 1h 20m 52s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/5f50087e5fc34661a8fa5530be7117a6 : SUCCESS in 1h 23m 08s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://zuul.opendev.org/t/openstack/build/17931f838c3c4dd69715ce102d1bf186 : FAILURE in 33m 21s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://zuul.opendev.org/t/openstack/build/dd7db54346964d0bb3167f85bec1bfad : FAILURE in 11m 12s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"ea195597e0fa028b06216cea663b6e12e824e5a1","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-19 02:41:43.000000000","message":"Patch Set 3:\n\nrecheck","accounts_in_message":[],"_revision_number":3},{"id":"96f6b7062e19e8ba36cda7a7b692273f3fc5062a","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-19 04:15:03.000000000","message":"Patch Set 3:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/32633999f6b3424190be4f48747c3f04 : SUCCESS in 1h 10m 18s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/799a5bdd407e4ae28ffb95c07152c65b : SUCCESS in 8m 10s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/b6ebe3e0df074017a4fb4413f14b8687 : SUCCESS in 8m 00s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/04ca735f1d1b4659845aafc082aa0f0c : SUCCESS in 4m 07s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/a2cc760a56854072b2fb4089e69077eb : SUCCESS in 7m 40s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/c66992149e904277af8d94e3b2fc048d : SUCCESS in 5m 46s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/9c3a2dd7982043778451ef55cb8a7738 : SUCCESS in 3m 25s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/5c80cb337a704a809cb243e6560c152d : SUCCESS in 3m 56s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/6216ab2bf237400b9e96b046ba72e030 : TIMED_OUT in 41m 00s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/68960d7d5143473ea7ddce7f91da7ae5 : TIMED_OUT in 40m 38s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/1cba5b4acc164066aba93c3308f59359 : SUCCESS in 6m 16s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/b3889857df754742829e2fb3e0efe46e : SUCCESS in 6m 02s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/b85b93f291584206b2a8dd394c3c241e : SUCCESS in 1h 18m 26s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/3040fbe6fcb8407c92b6e4d6b8085156 : SUCCESS in 1h 09m 34s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/7da465ff3c1845c49ae9ecb1a53d2add : SUCCESS in 1h 31m 16s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://zuul.opendev.org/t/openstack/build/1ec03288bb2746fb88a81b7182ed1b22 : FAILURE in 1h 04m 47s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://zuul.opendev.org/t/openstack/build/e770b6ce6e994a39a0ac37bbe8702c00 : FAILURE in 12m 13s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"e5b42942fba6d09526d6c4b359da52624abb5f80","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-20 04:24:37.000000000","message":"Patch Set 3:\n\nrecheck","accounts_in_message":[],"_revision_number":3},{"id":"c831979f4c59adc9d384209504420ff2af8e818d","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-20 05:55:21.000000000","message":"Patch Set 3:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/0fb8cdd4be784b06a2b879c65999a129 : SUCCESS in 1h 21m 01s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/806649c005514d199d96a791c1d76687 : SUCCESS in 6m 19s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/0e8b425b3bd64ded830d67a32be52873 : SUCCESS in 9m 40s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/e0f5678512cd4af19e71749e65ea8d1c : SUCCESS in 8m 39s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/5a1f522130df4e489ae4df42f7ba9430 : SUCCESS in 9m 01s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/bd76f218c3d040039adc6d0ce774b7a9 : SUCCESS in 10m 15s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/7bfa1910005346c9b9ca4186bc4a2c15 : SUCCESS in 5m 04s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/09b877a15aa944388091358199da4d6f : SUCCESS in 4m 42s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/60b369a1eb91407f91820f58120a3769 : TIMED_OUT in 40m 53s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/70b1ad23244841aca2a9d65f9b7e6d48 : TIMED_OUT in 40m 45s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/ae2619d8df94482d8c198f3699628292 : SUCCESS in 6m 39s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/867907a85abb4ba1b2a5fdf74cb2465d : FAILURE in 9m 44s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/8514132a204548f0ad946c1d83cacebe : SUCCESS in 1h 25m 42s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/dbe0fc92db374dcbb25c6e58937c7620 : SUCCESS in 1h 14m 54s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/436fa1057bf44fd294a4d332b00d53e6 : SUCCESS in 1h 28m 43s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://zuul.opendev.org/t/openstack/build/9a6307a61ca34b198201a79e8276e63d : FAILURE in 47m 42s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://zuul.opendev.org/t/openstack/build/720bcefc09d84304b41da06ae72f0c74 : FAILURE in 15m 07s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"c4c926d3763ac55e7394996567307a7f1f298743","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-20 17:14:57.000000000","message":"Patch Set 3:\n\nYou\u0027ll need to store the config in the Connection object and pass it to the Consumer constructor.  Try:\n\ndiff --git a/oslo_messaging/_drivers/impl_rabbit.py b/oslo_messaging/_drivers/impl_rabbit.py                                                      \nindex 679054cd..eff336aa 100644                                                                                                                   \n--- a/oslo_messaging/_drivers/impl_rabbit.py                                                                                                      \n+++ b/oslo_messaging/_drivers/impl_rabbit.py                                                                                                      \n@@ -238,11 +238,11 @@ class Consumer(object):                                                                                                     \n                                                                                                                                                  \n     def __init__(self, exchange_name, queue_name, routing_key, type, durable,                                                                    \n                  exchange_auto_delete, queue_auto_delete, callback,                                                                              \n-                 nowait\u003dFalse, rabbit_ha_queues\u003dNone, rabbit_queue_ttl\u003d0):                                                                       \n+                 nowait\u003dFalse, rabbit_ha_queues\u003dNone, rabbit_queue_ttl\u003d0,                                                                        \n+                 enable_cancel_on_failover\u003dFalse):                                                                                               \n         \"\"\"Init the Consumer class with the exchange_name, routing_key,                                                                          \n         type, durable auto_delete                                                                                                                \n         \"\"\"                                                                                                                                      \n-        driver_conf \u003d cfg.CONF.oslo_messaging_rabbit                                                                                             \n         self.queue_name \u003d queue_name                                                                                                             \n         self.exchange_name \u003d exchange_name                                                                                                       \n         self.routing_key \u003d routing_key                                                                                                           \n@@ -261,7 +261,7 @@ class Consumer(object):                                                                                                       \n             type\u003dtype,                                                                                                                           \n             durable\u003dself.durable,                                                                                                                \n             auto_delete\u003dself.exchange_auto_delete)                                                                                               \n-        self.enable_cancel_on_failover \u003d driver_conf.enable_cancel_on_failover                                                                   \n+        self.enable_cancel_on_failover \u003d enable_cancel_on_failover                                                                               \n                                                                                                                                                  \n     def declare(self, conn):                                                                                                                     \n         \"\"\"Re-declare the queue after a rabbit (re)connect.\"\"\"                                                                                   \n@@ -509,6 +509,7 @@ class Connection(object):                                                                                                     \n             global threading                                                                                                                     \n             threading \u003d stdlib_threading                                                                                                         \n         self.direct_mandatory_flag \u003d driver_conf.direct_mandatory_flag                                                                           \n+        self.enable_cancel_on_failover \u003d driver_conf.enable_cancel_on_failover                                                                   \n                                                                                                                                                  \n         if self.ssl:                                                                                                                             \n             self.ssl_version \u003d driver_conf.ssl_version                                                                                           \n@@ -1139,7 +1140,8 @@ class Connection(object):                                                                                                   \n                             queue_auto_delete\u003dFalse,                                                                                             \n                             callback\u003dcallback,                                                                                                   \n                             rabbit_ha_queues\u003dself.rabbit_ha_queues,                                                                              \n-                            rabbit_queue_ttl\u003dself.rabbit_transient_queues_ttl)                                                                   \n+                            rabbit_queue_ttl\u003dself.rabbit_transient_queues_ttl,                                                                   \n+                            enable_cancel_on_failover\u003dself.enable_cancel_on_failover)                                                            \n                                                                                                                                                  \n         self.declare_consumer(consumer)                                                                                                          \n                                                                                                                                                  \n@@ -1154,7 +1156,8 @@ class Connection(object):                                                                                                   \n                             exchange_auto_delete\u003dself.amqp_auto_delete,                                                                          \n                             queue_auto_delete\u003dself.amqp_auto_delete,                                                                             \n                             callback\u003dcallback,                                                                                                   \n-                            rabbit_ha_queues\u003dself.rabbit_ha_queues)                                                                              \n+                            rabbit_ha_queues\u003dself.rabbit_ha_queues,                                                                              \n+                            enable_cancel_on_failover\u003dself.enable_cancel_on_failover)                                                            \n                                                                                                                                                  \n         self.declare_consumer(consumer)                                                                                                          \n                                                                                                                                                  \n@@ -1174,7 +1177,8 @@ class Connection(object):                                                                                                   \n                             queue_auto_delete\u003dFalse,                                                                                             \n                             callback\u003dcallback,\n                             rabbit_ha_queues\u003dself.rabbit_ha_queues,                                                                              \n-                            rabbit_queue_ttl\u003dself.rabbit_transient_queues_ttl)                                                                   \n+                            rabbit_queue_ttl\u003dself.rabbit_transient_queues_ttl,                                                                   \n+                            enable_cancel_on_failover\u003dself.enable_cancel_on_failover)                                                            \n                                                                                                                                                  \n         self.declare_consumer(consumer)","accounts_in_message":[],"_revision_number":3},{"id":"7c2459dde829c79ebfc980d908f529a9eb668c95","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-20 17:30:48.000000000","message":"Patch Set 3:\n\nRegarding unit tests:\n\nWe should probably enable this feature during the rabbitmq failover tests in tox.\n\nThis can be done by adding the config option to the test in oslo_messaging/tests/functional/test_rabbitmq.py:\n\n        # NOTE(sileht): Allow only one response at a time, to                                                                                     \n        # have only one tcp connection for reply and ensure it will failover                                                                      \n        # correctly                                                                                                                               \n        self.config(heartbeat_timeout_threshold\u003d1,                                                                                                \n                    rpc_conn_pool_size\u003d1,                                                                                                         \n                    kombu_reconnect_delay\u003d0,                                                                                                      \n                    rabbit_retry_interval\u003d0,                                                                                                      \n                    rabbit_retry_backoff\u003d0,                                                                                                       \n+                    enable_cancel_on_failover\u003dTrue,                                                                                               \n                    group\u003d\u0027oslo_messaging_rabbit\u0027)","accounts_in_message":[],"_revision_number":3},{"id":"7fd3be52842e77586b12342464669da82d978826","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-22 12:35:15.000000000","message":"Patch Set 3:\n\nrecheck","accounts_in_message":[],"_revision_number":3},{"id":"ac1e73cd0ffca1f9f3456b12ef907e39e3ac43b2","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-22 16:07:52.000000000","message":"Patch Set 3:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/dd822aaf386747f99c0c8be1f3cb4788 : FAILURE in 1h 32m 25s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/3e9fc823d7c34251bfa997e0657db283 : SUCCESS in 6m 30s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/72e8964d60c94f30bc72dbba29501610 : SUCCESS in 7m 46s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/65b78f6f08bc483884c31a519ced9e9e : SUCCESS in 5m 16s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/f1e96d0920cf40aea8e7b3ccc5434d7f : SUCCESS in 8m 04s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/065c0b07c1c44b1daf0e8620ece7d803 : SUCCESS in 6m 32s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/8506a45b14b14f12a6873561d267ade1 : SUCCESS in 4m 41s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/f5c8acf2830e4661acd663342e406702 : SUCCESS in 3m 52s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/7ad9f474c5e2444088f2fe7dffe61a06 : TIMED_OUT in 41m 05s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/1927d64cc1f54a3bbae811fd456f33c0 : TIMED_OUT in 41m 36s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/3e9b36afedde422e84379f5486176932 : SUCCESS in 7m 16s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/657f3298269042ed8aad08cc93144f97 : FAILURE in 8m 02s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/c9f9e973b17c41418c3cd5e23d6b2d5b : POST_FAILURE in 1h 46m 58s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/7e85855841774b619b6c5904419f9b8e : SUCCESS in 1h 32m 36s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/d1d36fc64737478f8ee92a57c82aed42 : SUCCESS in 1h 38m 18s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/1d8f9ff0ab3645e599573fe679c58622 : SUCCESS in 1h 07m 53s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/6f3e604dddb04b358ced2cbe32d2f6d1 : SUCCESS in 1h 30m 56s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"206f77eafa432fad8329548b5cac22972ce62692","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-28 15:11:48.000000000","message":"Patch Set 3: Code-Review-1\n\nThese tests will not pass - the configuration has to be read at Connection creation, not on Consumer creation.\n\nPlease see my previous comment, that should get these tests to pass so we can submit this fix.  thanks.","accounts_in_message":[],"_revision_number":3},{"id":"1b7e4c3666654dfe68be89d012648fed8b19d5aa","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-29 00:04:34.000000000","message":"Patch Set 3:\n\nthank you, I will fix that.","accounts_in_message":[],"_revision_number":3},{"id":"4ab55fd7f6b126a75e45cedf8c0bdc117651c1a7","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-30 15:10:43.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"4e6f45207c95bd913a6170e5c324ec5df3d7dc71","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-30 15:14:28.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"5ab575ecfdca6b00f0f22c2601aaa0a2156302ec","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-30 17:06:52.000000000","message":"Patch Set 5: Verified-1\n\n(3 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/aac40e391e574cd59a4c73a0bc2fa0a1 : SUCCESS in 1h 31m 28s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/585960af1701438d860d1052a17d6292 : SUCCESS in 6m 18s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/5a74820a311142218736dc1d087e395f : SUCCESS in 4m 59s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2e49127d5d68445ba69c8ac475c09bda : FAILURE in 6m 50s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/8d7f90320e3d4f29953964571e420245 : SUCCESS in 5m 54s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/5e9c74ee4edd402f8ff578158b7b2109 : SUCCESS in 6m 18s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/defa97072b0c4a9790aa064c49e47b23 : SUCCESS in 4m 23s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/c6089385ccf6464bb9983fde84d5bbee : SUCCESS in 3m 33s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/9be7fd7773a14e95bb8259e1569d7144 : SUCCESS in 5m 23s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/db350021e2d741e19448b2afdc399035 : FAILURE in 6m 39s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/0d2474159bab4d4c99f44103cfbfd596 : SUCCESS in 6m 18s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/8fb7c525eb6843b282d27aa1cff70b8f : FAILURE in 6m 48s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/b10ae629bd1e4363bf8a91c77be553aa : SUCCESS in 1h 26m 51s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/bc33cdb5483e40f89c3e17beaf25b0d3 : SUCCESS in 1h 40m 47s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/c65b7150993b4fa3a589d1b81616c58c : SUCCESS in 1h 30m 40s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/6870c5cb5ea941619300602f2372668f : SUCCESS in 45m 20s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/60f4822b0cc84e078e7d95b67aacf760 : SUCCESS in 1h 20m 26s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"807f220dc8b7437934bf6b01071b9d3776c2c24d","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-30 22:05:49.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"dc44b4c5e1c75dc0f3453795206f0007b4313795","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-30 23:52:22.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/3c397a653af94d8f9e23b878239c1d6a : SUCCESS in 1h 41m 02s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/8276d5d7351b4e149d1b97f01add2a99 : SUCCESS in 6m 40s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/a0617f948e4c4968bd97c5e4d127400a : SUCCESS in 6m 30s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d70e8bda1526443ba52493c15dc863a1 : SUCCESS in 7m 20s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/da7c415038c64d2e838dc9d75c04400e : SUCCESS in 5m 50s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/fec616789e4c483ebfea0318582bffa8 : SUCCESS in 5m 47s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/d7d8e9215f854b1aa06cfd185d1fc39c : SUCCESS in 3m 53s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/5e84369a47674596ac003b07eeb164eb : SUCCESS in 3m 24s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/d93593a88d1845eda11597161bcb210d : SUCCESS in 6m 05s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/811d128d84ec42479cb421ca778d4602 : FAILURE in 9m 02s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/d2abbad65f8e446889c1af713b90b02b : SUCCESS in 6m 37s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/f478c99e16734a45a552d6605363e58c : FAILURE in 6m 36s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/bf105bc217b5447281cc15c7b8d97b45 : SUCCESS in 1h 32m 17s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/045db7050c1d4c378107640041e53c29 : SUCCESS in 58m 54s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/c9366a169a854b10a1358da7d8c57f8d : SUCCESS in 1h 28m 57s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/294669b240e1427d8bfb4c2e65156837 : SUCCESS in 40m 50s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/aca6014aeb3f4884a8d6bafbb4478fcc : SUCCESS in 1h 06m 06s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"ae80860e74de9241fde3d5d22e1d7622cb52d844","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2020-07-31 08:28:57.000000000","message":"Patch Set 6: Code-Review+2","accounts_in_message":[],"_revision_number":6},{"id":"8c6fb7bc24e75a7fdd0e04ac7bc4f1ec5333772d","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-31 17:29:59.000000000","message":"Patch Set 6: Workflow+1","accounts_in_message":[],"_revision_number":6},{"id":"1d1a6a32544faaebeca6bcce1a1a5ca0990929d1","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-31 17:30:11.000000000","message":"Patch Set 6: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":6},{"id":"e7e8b928fa182579efed493d2decf1a5d5a34730","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2020-07-31 17:30:16.000000000","message":"Patch Set 6: Code-Review+2","accounts_in_message":[],"_revision_number":6},{"id":"bbca3a1053ccf71e425aff56febe3b269f352053","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-31 19:11:48.000000000","message":"Patch Set 6: Verified-2\n\nBuild failed (gate pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/1ecb4433499242dc9c434cb418487b89 : SUCCESS in 1h 10m 48s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/23608bf534344a7d8aafb5cecda0a740 : SUCCESS in 5m 03s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/9a0232f5fba74dc1bd6aae01d992cdda : SUCCESS in 4m 13s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/68cc48ab3b544dcdbc8003c7c3458702 : SUCCESS in 8m 20s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/917b1844e4ce4d2cb919355985462120 : SUCCESS in 7m 46s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6dad3e4835a743f3a1991ea0d39b9edf : SUCCESS in 5m 16s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/9f7b673b295149ac84375fbc4074fb94 : SUCCESS in 3m 30s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/f6836f092f504823ba7bed76afb7aab2 : SUCCESS in 7m 31s\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/056e2f91d5644c61a62fdc97a4286d0f : FAILURE in 1h 36m 25s","accounts_in_message":[],"_revision_number":6},{"id":"74f416028b21f3821ebd85290c8114a705c4e1f3","author":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"date":"2020-07-31 22:07:40.000000000","message":"Patch Set 6:\n\nrecheck","accounts_in_message":[],"_revision_number":6},{"id":"f5f027574b4889bf7529c079684afb42f4d58b14","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-08-01 00:00:18.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/e162174c0d07405e8dc525cb4fb80f8d : SUCCESS in 1h 08m 44s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/940610cfd650421498473ca88d4b5c0d : SUCCESS in 6m 27s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/1f9fcdc8bf0148f18a436afd948a95ac : SUCCESS in 5m 55s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/58332977901940cc97566076bac97c95 : SUCCESS in 5m 03s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/21fbe6b8f4f6451a8090351d203a3cfd : SUCCESS in 5m 47s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/f90e02845c6c4db9be71c9c9156eeaa6 : SUCCESS in 7m 31s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4f1660c6c3bf43a39fbe8d4fec6e1de0 : SUCCESS in 4m 35s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/0fa2eedae71b4bce8b6aab768a85ee5c : SUCCESS in 4m 30s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/b211a87c038245f492b95d4666fc175f : SUCCESS in 5m 08s\n- oslo.messaging-tox-py36-func-scenario02 https://zuul.opendev.org/t/openstack/build/c45aa4f93dfc4060b9ac873354f6c255 : FAILURE in 5m 39s (non-voting)\n- oslo.messaging-tox-py36-func-scenario03 https://zuul.opendev.org/t/openstack/build/46cb4b678b574fdfbae650dbd759ff76 : SUCCESS in 6m 01s (non-voting)\n- oslo.messaging-tox-py36-func-scenario04 https://zuul.opendev.org/t/openstack/build/e815373881b94d159a5a02f95b0c0df6 : FAILURE in 5m 33s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/c4bb7e1081da422681b03bd339e9da4e : SUCCESS in 1h 43m 54s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://zuul.opendev.org/t/openstack/build/4657b39d71964d40aa62851795f6d069 : SUCCESS in 1h 23m 11s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/942f534e53ea41edb18f32a59840c333 : SUCCESS in 1h 48m 49s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/8f2f22e631c947d892d0f26ae4619f87 : SUCCESS in 1h 00m 59s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/1ce51cf03a324b949db2ca63974c6afe : SUCCESS in 52m 23s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"57fe064b571f11b4ef574bbe1d0fe7ed21fe4fb7","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-08-01 00:00:30.000000000","message":"Patch Set 6: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":6},{"id":"111526b5876f925c28a6de7376b280522ed726d3","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-08-01 01:21:42.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":6},{"id":"799471ca78db3756465d8ad14751b6302064ac3f","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-08-01 01:21:42.000000000","message":"Patch Set 6: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/31c6b5efa64a4648b0e1c320d9346ee7 : SUCCESS in 1h 20m 26s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/d8e78465a7c041ada8ddb4d60676c35f : SUCCESS in 6m 39s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/200d83e1eb72463485c581bf580dc8f5 : SUCCESS in 4m 42s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/636fb1e0ea244e378e3fbd60fee805c8 : SUCCESS in 7m 07s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/2cec217b220d4d358cea9f9d19104e91 : SUCCESS in 5m 57s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ea82f6e2fe9440e180575a0fd33eeacc : SUCCESS in 3m 43s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/4b35cd88e6174fb09b726f758b0f88f5 : SUCCESS in 2m 57s\n- oslo.messaging-tox-py36-func-scenario01 https://zuul.opendev.org/t/openstack/build/9270ca315d5f4880bd55fcce84cad0b7 : SUCCESS in 5m 10s\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/f2e85140030e4e68b3f91b7918274cb4 : SUCCESS in 56m 23s","accounts_in_message":[],"_revision_number":6},{"id":"816fdcccf5845c3d35297523ea5fe617497c1fc1","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-08-01 01:22:56.000000000","message":"Patch Set 6:\n\nBuild succeeded (promote pipeline).\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/0bb26134715f49608296d0c749ff38a5 : SUCCESS in 47s\n- promote-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cd7a5847dfb5416288c79067f09ec020 : SUCCESS in 51s","accounts_in_message":[],"_revision_number":6}],"current_revision_number":6,"current_revision":"196fa877a90d7eb0f82ec9e1c194eef3f98fc0b1","revisions":{"918f1c73405497c52269041955ba669a2b3be762":{"kind":"REWORK","_number":1,"created":"2020-07-03 07:55:55.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/1"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges to avoid excessive amounts of exchange not found messages. But it does not actually solve the problem because reply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances seems rabbitmq consumer does not receive basic cancel signal when queue is already gone. This might due to rabbitmq try to restart consumer when queue is down (for example when split brain). In such cases, it might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover is not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup, I could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/918f1c73405497c52269041955ba669a2b3be762"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/918f1c73405497c52269041955ba669a2b3be762"}]},"branch":"refs/heads/master"},"3f06ab9fbb0e6fcead4ea56f00d6b3de08243b18":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2020-07-03 08:04:01.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/2"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:59:37.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges\nto avoid excessive amounts of exchange not found messages.\nBut it does not actually solve the problem because\nreply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances\nseems rabbitmq consumer does not receive basic cancel\nsignal when queue is already gone. This might due to\nrabbitmq try to restart consumer when queue is down\n(for example when split brain). In such cases,\nit might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover\nis not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894,\nhttps://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup,\nI could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/3f06ab9fbb0e6fcead4ea56f00d6b3de08243b18"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/3f06ab9fbb0e6fcead4ea56f00d6b3de08243b18"}]},"branch":"refs/heads/master"},"d21b6fc13c3a55346b6e50a199d0cae437f96056":{"kind":"REWORK","_number":3,"created":"2020-07-19 00:20:12.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/3"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"ushen","email":"yshxxsjt715@gmail.com","date":"2020-07-19 00:19:37.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges\nto avoid excessive amounts of exchange not found messages.\nBut it does not actually solve the problem because\nreply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances\nseems rabbitmq consumer does not receive basic cancel\nsignal when queue is already gone. This might due to\nrabbitmq try to restart consumer when queue is down\n(for example when split brain). In such cases,\nit might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover\nis not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894,\nhttps://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup,\nI could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d21b6fc13c3a55346b6e50a199d0cae437f96056"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d21b6fc13c3a55346b6e50a199d0cae437f96056"}]},"branch":"refs/heads/master"},"78ac1496fb7062cfe3e9bb5248fdd3cb3e69297c":{"kind":"REWORK","_number":4,"created":"2020-07-30 15:10:43.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/4"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"ushen","email":"yshxxsjt715@gmail.com","date":"2020-07-30 15:01:27.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges\nto avoid excessive amounts of exchange not found messages.\nBut it does not actually solve the problem because\nreply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances\nseems rabbitmq consumer does not receive basic cancel\nsignal when queue is already gone. This might due to\nrabbitmq try to restart consumer when queue is down\n(for example when split brain). In such cases,\nit might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover\nis not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894,\nhttps://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup,\nI could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/78ac1496fb7062cfe3e9bb5248fdd3cb3e69297c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/78ac1496fb7062cfe3e9bb5248fdd3cb3e69297c"}]},"branch":"refs/heads/master"},"647f6e23def8b6710db6a0af0f111b83fdda17c1":{"kind":"REWORK","_number":5,"created":"2020-07-30 15:14:28.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/5"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"ushen","email":"yshxxsjt715@gmail.com","date":"2020-07-30 15:13:37.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges\nto avoid excessive amounts of exchange not found messages.\nBut it does not actually solve the problem because\nreply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances\nseems rabbitmq consumer does not receive basic cancel\nsignal when queue is already gone. This might due to\nrabbitmq try to restart consumer when queue is down\n(for example when split brain). In such cases,\nit might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover\nis not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894,\nhttps://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup,\nI could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/647f6e23def8b6710db6a0af0f111b83fdda17c1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/647f6e23def8b6710db6a0af0f111b83fdda17c1"}]},"branch":"refs/heads/master"},"196fa877a90d7eb0f82ec9e1c194eef3f98fc0b1":{"kind":"REWORK","_number":6,"created":"2020-07-30 22:05:49.000000000","uploader":{"_account_id":29071,"name":"norman shen","email":"yshxxsjt715@gmail.com","username":"ushen"},"ref":"refs/changes/75/739175/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/75/739175/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/75/739175/6"}}},"commit":{"parents":[{"commit":"7e406c312a6514e7ae377edb52b9e02b5bf37a7d","subject":"Simplify tools/test-setup.sh","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7e406c312a6514e7ae377edb52b9e02b5bf37a7d"}]}],"author":{"name":"shenjiatong","email":"yshxxsjt715@gmail.com","date":"2020-07-03 07:51:21.000000000","tz":480},"committer":{"name":"ushen","email":"yshxxsjt715@gmail.com","date":"2020-07-30 22:05:16.000000000","tz":480},"subject":"Cancel consumer if queue down","message":"Cancel consumer if queue down\n\nPreviously, we have switched to use default exchanges\nto avoid excessive amounts of exchange not found messages.\nBut it does not actually solve the problem because\nreply_* queue is already gone and agent will not receive callbacks.\n\nafter some debugging, I found under some circumstances\nseems rabbitmq consumer does not receive basic cancel\nsignal when queue is already gone. This might due to\nrabbitmq try to restart consumer when queue is down\n(for example when split brain). In such cases,\nit might be better to fail early.\n\nby reading the code, seems like x-cancel-on-ha-failover\nis not dedicated to mirror queues only, https://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1894,\nhttps://github.com/rabbitmq/rabbitmq-server/blob/master/src/rabbit_channel.erl#L1926.\n\nBy failing early, in my own test setup,\nI could solve a certain case of exchange not found problem.\n\nChange-Id: I2ae53340783e4044dab58035bc0992dc08145b53\nRelated-bug: #1789177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/196fa877a90d7eb0f82ec9e1c194eef3f98fc0b1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/196fa877a90d7eb0f82ec9e1c194eef3f98fc0b1"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}