)]}'
{"id":"openstack%2Foslo.messaging~993316","triplet_id":"openstack%2Foslo.messaging~master~I8b0d949c148e01366cf38789a009c4aef7b14843","project":"openstack/oslo.messaging","branch":"master","topic":"bp/oslo-http-driver","attention_set":{"10273":{"account":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"last_update":"2026-06-15 11:40:48.000000000","reason":"A robot voted negatively on a label"}},"removed_from_attention_set":{},"hashtags":[],"change_id":"I8b0d949c148e01366cf38789a009c4aef7b14843","subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","status":"NEW","created":"2026-06-15 09:23:39.000000000","updated":"2026-07-17 07:23:03.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":5,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"503dc6ab1b919165df6efa637a60e39fb163a95e","_number":993316,"virtual_id_number":993316,"owner":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":1,"date":"2026-07-17 07:23:03.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Backport-Candidate":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do Not Backport","-1":"Not A Backport Candidate"," 0":"Backport Review Needed","+1":"Proposed Backport","+2":"Should Backport"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2026-06-15 11:40:48.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"6c43c20752520bcbf09dcef4e12ba64b0a937b65","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-15 09:23:39.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"c9fd52595ee2ae720740113546a670175055e015","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-15 11:40:48.000000000","message":"Patch Set 1: Verified-1\n\n(3 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/2d9d14efada24adcac747ce4fa70bd17\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/29e6b9667fff436d90725c00467cf666 : SUCCESS in 2m 37s\n- requirements-check https://zuul.opendev.org/t/openstack/build/0d7b2bc8f87340c2af4c417fb18bd9a5 : SUCCESS in 2m 06s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/b8d69193355e4603aabfd5d9b1024072 : POST_FAILURE in 1h 35m 42s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/4839e87e34734faebdb32457b56ab39b : SUCCESS in 4m 23s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/aa251a2517394776b86dfde00730801e : FAILURE in 3m 26s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/0392af4f70904300a5e0f4111b396929 : SUCCESS in 4m 00s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/bb8661fd110142e39da2144661145a26 : SUCCESS in 3m 13s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/7d56cdd159e9431c961a30c4c4def348 : SUCCESS in 6m 16s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b4708a98267f4fb5b40f074a2b4e5b87 : SUCCESS in 2m 51s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/c251cf0ad89943fbb74c2c8db068978d : SUCCESS in 3m 39s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/caad62c14d1b47e3a6a37c090c16a1a8 : SUCCESS in 5m 25s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/e95fd8badec146e09d301f22e0fe2943 : FAILURE in 13m 48s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/79fc0d1b1eb2430db1058ac536791000 : FAILURE in 2m 46s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/dabd8e24f3f94e63875bbc8cf7b0101b : SUCCESS in 1h 19m 38s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/3a38f96b28dc4cfcaf011336b95aa2ca : SUCCESS in 2h 07m 08s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/ebfe37f1bc6645368386ee1565f15d4b : SUCCESS in 1h 05m 37s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/3ea336b07b3a48bb8c93d0705470af26 : POST_FAILURE in 1h 29m 59s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"3d479ebe14179c2c73972b2c910712f77d46de98","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-16 08:49:32.000000000","message":"Uploaded patch set 2.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":2},{"id":"280df565e56b0440f471f99c23cd0316612b87e5","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-16 11:00:45.000000000","message":"Patch Set 2: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/3178a989530b43f3ae7a29c80aa56f02\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/d37c212bc2ad477eaff397b672df5f30 : SUCCESS in 3m 05s\n- requirements-check https://zuul.opendev.org/t/openstack/build/766a87f86aeb4286a8cb9297035d686c : SUCCESS in 2m 53s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/a429f45542bb4408b7e8f4b33bcc23c5 : SUCCESS in 1h 41m 16s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/a9a278f3fbc64df889a96e13fe7f8251 : SUCCESS in 5m 28s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/3d1ce317cc894d518a423da7638f0d34 : FAILURE in 3m 37s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/f22733dd20234f85985b31d4c2462152 : SUCCESS in 4m 23s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/096591698c434b718e105eb2953c898c : SUCCESS in 3m 52s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/604c9984ac394c75836e862c61a7b692 : SUCCESS in 7m 25s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b1fb3711ce574c218660b0a7b02d2763 : SUCCESS in 4m 32s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/191e7bf1fd8b4eaf8f242f249d644700 : SUCCESS in 4m 58s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/41769f443ce2403386cbd9f67a5fed5f : SUCCESS in 6m 14s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/8a531b50ccee4940ac9ff004bc90319f : FAILURE in 13m 58s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/b15ae835c27c42fea40bf46b0a534ca6 : FAILURE in 3m 31s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/fb600dc0c3eb48ffa15417c7285f8943 : SUCCESS in 1h 44m 24s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/0f492c04b09b47fdb80887129a72796b : SUCCESS in 1h 56m 17s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/b6fd9f27d0d54c3aa639a1f28e6c405c : SUCCESS in 1h 06m 41s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/31dc19a2249746c2a61e139a2a2fba34 : SUCCESS in 1h 18m 12s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"ff8f7876f0e2e0834a235976a1c3154b093931af","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-17 13:29:16.000000000","message":"Uploaded patch set 3.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":3},{"id":"07fda0850c7c00ed6f0d963e0abd94a6bfa1f6f6","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-17 15:17:39.000000000","message":"Patch Set 3: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/0c369cef6c0e4adfa34e2841e185ddcc\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/de8fecbe39cd4652952789615957c1f7 : SUCCESS in 3m 18s\n- requirements-check https://zuul.opendev.org/t/openstack/build/9ea87f551772482d9db70eddca40d0eb : SUCCESS in 2m 36s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/6c26c6190f504004a505e3d6505e0712 : SUCCESS in 58m 26s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/6f5604f320484d5994eb8e99ba6fb1f6 : FAILURE in 4m 05s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/e8ab86209cf948f3a49c97e43c80e4af : FAILURE in 3m 48s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/74803d10fc824bcbb33bfe523c78fbdb : FAILURE in 6m 19s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/b791e4df393449359b9ba60d9f3b3f83 : FAILURE in 3m 04s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/2e617d1627de4c8da9cdc26fa2aab630 : FAILURE in 8m 38s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e24bd3cfcfd447749a79c8dcb5518d0d : SUCCESS in 4m 24s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/8b0bb733022e4588af23a113ce1b2485 : SUCCESS in 4m 48s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/2a50b921e7174463ba7229e268421318 : SUCCESS in 5m 22s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/d50d0ff0c9474e1f87546b7caecb40c9 : FAILURE in 4m 49s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/0ae7405cd380407abee813a90225f7a8 : FAILURE in 2m 01s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/728637d3a108405c8c439617d2fecfda : SUCCESS in 1h 42m 37s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/b1863ca83793485ba004de98dc0f1457 : SUCCESS in 1h 03m 44s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/d7f87d81af8448649e42f025dc87fa2e : SUCCESS in 1h 04m 52s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/1ce052cd16884a99b0f581ac3b82ec47 : SUCCESS in 1h 22m 25s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"21033553a86640615bb72aac6d34fc93f682b032","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-23 02:12:14.000000000","message":"Uploaded patch set 4.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":4},{"id":"df66ddeb45cfeebe4f58a1174ad5f2c07b829ec6","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-23 04:47:21.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/e75a61e0490e4593b2554cec4cd66283\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/254b086c75c74f55b5a490d74a1f278b : SUCCESS in 3m 18s\n- requirements-check https://zuul.opendev.org/t/openstack/build/a3725706342c435a851779a21d445ae1 : FAILURE in 2m 58s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/0e559f768a5348b1be6551100e4af3d7 : SUCCESS in 1h 16m 39s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/9bd504fae4d54dbe9835ef519d1c11da : SUCCESS in 4m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/1cffd1c7f0884d9bb1c1965555f0a246 : SUCCESS in 4m 14s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/5fa1ffac7736486785f0383a8d3fbc7e : SUCCESS in 4m 34s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/b512c52978704b49a15ec536d2d25a0b : SUCCESS in 5m 26s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/c3b2e03c8ee24c42b43b7c40f67170e5 : SUCCESS in 4m 42s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e24cabc370ac4cf0a380c0a7e78d28c6 : SUCCESS in 3m 05s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cd736fa7a3014b258fad65b08fbfcefd : SUCCESS in 5m 09s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/e7931631e2334bcba4941fad4fa895dd : SUCCESS in 6m 05s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/f4d33b7e7e6d4a8697b596e738025600 : FAILURE in 4m 55s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/ad220b75e9694b5893eb6ea167bf31c2 : FAILURE in 3m 01s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/9adda8b32a72403eb811013d590122cd : SUCCESS in 1h 21m 10s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/767767707ead489fa094c82bb03f2f1e : SUCCESS in 1h 48m 04s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/2ba83e5e34f34708b6885705d3232ae0 : SUCCESS in 1h 04m 43s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/32a207a60b6e4c06b5c5b553790d4d3e : SUCCESS in 1h 07m 49s (non-voting)","accounts_in_message":[],"_revision_number":4},{"id":"5b48b6a75ae729a77d02cea37a165b63cafab37e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-23 19:22:34.000000000","message":"Uploaded patch set 5.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":5},{"id":"0ecedfc28e2580641035e2df7d075f9362c9ba33","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-23 21:24:24.000000000","message":"Patch Set 5: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/22f8d8cda2d14555b5a624e0dd9c6e96\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/a9277fe568244813addb7b7b75627beb : SUCCESS in 2m 49s\n- requirements-check https://zuul.opendev.org/t/openstack/build/3fe31b3b2ec04ab3bb250ab3593cc842 : SUCCESS in 2m 23s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/6b94b6f366cc488a876431ea1a765f60 : SUCCESS in 51m 47s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/d28689adbae942e48296635a2b3ddf95 : SUCCESS in 4m 28s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/639b99be6d1a49569d81a6788af15336 : SUCCESS in 3m 39s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/cf73891c61c54ede81b1a1b85562f887 : SUCCESS in 4m 03s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/9f78e1f2cae6428888845bf8171dfb56 : SUCCESS in 3m 51s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/e5f7200f1cfd472396683e3a20eb9ca3 : SUCCESS in 8m 17s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c81855c79e544305be7fac8faaf74222 : SUCCESS in 3m 38s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/f60279fba3924cc6bb9904faa6020521 : SUCCESS in 4m 26s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/be01ec6744124c31aeb9b645fa0140cb : SUCCESS in 4m 34s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/05019bfe3d6a44af9cd8c6acfd728e81 : FAILURE in 4m 55s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/2049213223d74f2f9056c78f50e3f662 : FAILURE in 1m 58s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/ecce7294433c49609f6e8f0787c24160 : SUCCESS in 1h 55m 05s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/7cdedf12580f4ca3bf7a960b3be47ff0 : SUCCESS in 1h 52m 21s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/1a5bc89f77ff4b1c8b7d6d5598861dcd : SUCCESS in 1h 05m 38s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/644adada89a8495786dc9c8306107fb9 : SUCCESS in 1h 20m 29s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"152a94fd0903ec8a804c6b629de6d011d9006b4b","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-25 09:09:08.000000000","message":"Uploaded patch set 6.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":6},{"id":"ca40192d022628b2d281e42f3ad40b854b5f642e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-25 10:44:00.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/20fb06987fa14df19114ba6dee786f75\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/f4907f02d3314df7bbd6d425b9912c55 : SUCCESS in 2m 30s\n- requirements-check https://zuul.opendev.org/t/openstack/build/d41b842218c140fa8bf5747562004d9b : SUCCESS in 3m 17s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/7ed9c0e3c722402389483197f9ae05f1 : SUCCESS in 49m 41s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/3c40dfc9fad742898371ed53219314bc : SUCCESS in 4m 41s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/749b814516744c078cab32702fbe8ec0 : SUCCESS in 3m 20s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/cc66e2fd8ab043eda65e7e2cfbf64cfe : SUCCESS in 3m 15s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/33e406b0e1884f89b7aca6f172fdb26d : SUCCESS in 4m 42s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/3793d8f5fd514cbdb7747c352d462fec : SUCCESS in 6m 46s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/2b0130e1c3274a11ba6aaf961aa904aa : FAILURE in 5m 49s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/2168f4cc211e4ad8b3f6bc29a2d77713 : SUCCESS in 3m 54s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/af1d28a86f8c4e1fa8fa17cc6cd1a1cd : SUCCESS in 2m 30s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/4dc2f61132824ec2a3b79df3eead2245 : SUCCESS in 5m 30s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/dc878552e42b41e286d44675c4572ab1 : FAILURE in 18m 07s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/7fcc3b0aa08a468b927c1ecf79ee86f9 : FAILURE in 4m 51s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/45f62b52c6a74394b1cf236a9c874493 : SUCCESS in 1h 11m 17s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/628976db577c4b51aa0bc44bd82ed5c7 : SUCCESS in 1h 29m 47s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/1e9e929ea19c45768946afb17c9edaa3 : SUCCESS in 54m 25s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/31f62b754e134d84bdccc40ff1421966 : SUCCESS in 1h 20m 49s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"33108e22eaa16f3e393d12d4a0e50fa4674832af","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-26 00:41:12.000000000","message":"Uploaded patch set 7.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":7},{"id":"78e82a6a23aae3562a032084a3dc751a14b7b365","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-26 02:30:05.000000000","message":"Patch Set 7: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/1bc60ca4007945a5af476a89d2fe6bd0\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/0cd17efdab7f49b3b2f272cfdb3660d2 : SUCCESS in 1m 55s\n- requirements-check https://zuul.opendev.org/t/openstack/build/8f5aedc867a145899766f00ce45ce259 : SUCCESS in 1m 55s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/5c047fce86f9489381dc34a75d8a6b39 : SUCCESS in 44m 08s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/41a9994db53240c2b32218c25a49a67e : SUCCESS in 5m 50s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/5fce9a85c1ef4402b1d655d25d63cfd8 : SUCCESS in 2m 13s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/234f77288a6f4362ad7c5ceff5caba7b : SUCCESS in 3m 55s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/ef14b7eb6ad34545a0c865c32a11fe40 : SUCCESS in 2m 50s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/b40d59b861c14fa4bd73c00d6ff06584 : SUCCESS in 4m 59s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/627600f7947e42d6a1a35e7d1520d72a : FAILURE in 5m 35s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3af202bac634485bac1fca95dd7bd582 : SUCCESS in 2m 36s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/3a82482de2e2473d920222ca1417d8d5 : SUCCESS in 2m 45s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/bceb23512eba4151b61e91deec9a8b58 : SUCCESS in 4m 28s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/11fda4311cb945008f7ce082db4449cb : FAILURE in 4m 12s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/72ec871a13db4da2bb540a86efee0af6 : FAILURE in 3m 21s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/d920b797f5da4cedab5f7b6fb8ff4899 : SUCCESS in 1h 03m 47s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/233beecd9d5a4a8284e95897b8baaa7a : SUCCESS in 1h 41m 41s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/f8f0a935f7ad43a18df626795cf3778b : SUCCESS in 1h 06m 09s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/774383287746409ab6729b95b2939073 : SUCCESS in 1h 15m 17s (non-voting)","accounts_in_message":[],"_revision_number":7},{"id":"fdd8f687685840b3afc149cecb94f599ac1a1cce","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-26 05:42:16.000000000","message":"Uploaded patch set 8.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":8},{"id":"a62a1af466d69114b5fd8db0e1cf404001dcc086","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-26 07:08:22.000000000","message":"Patch Set 8: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/5993baccf5f7424db66e8c65c7488349\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/e1d519e80675458887f42455189a9585 : SUCCESS in 3m 04s\n- requirements-check https://zuul.opendev.org/t/openstack/build/a3352b5132cb44ae8609ef4ff3e5dcfa : SUCCESS in 2m 26s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/8ffcf273638d4c6f8d239910a6eb94aa : SUCCESS in 1h 20m 24s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/4f1915294979446293636af5474a5af8 : SUCCESS in 2m 57s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/17023ea6211c46d89f5e0c00973fe386 : SUCCESS in 4m 56s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/4e39244d7a2a406990c36c79f5dc03de : SUCCESS in 3m 00s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/4ae3f247c02941729616b01a35718bd3 : SUCCESS in 3m 56s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/a1987754f3bf476f87702a2a2e8fd2c3 : SUCCESS in 6m 27s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/049cbd6591494c83a7f1066009c8da10 : FAILURE in 3m 54s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f8894647f1364aaf91fa83040b2d4c8c : SUCCESS in 2m 20s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/d06b7f22c65a4bc7934e57299c4c7d7e : SUCCESS in 4m 20s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/8157b07394454578a14db555a315dac2 : SUCCESS in 3m 38s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/bf564bd6107e4c47ad7660e4076497ac : FAILURE in 4m 41s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/8b742e3b303b46c58997d1c82fa6182c : FAILURE in 2m 25s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/3bd63c2eef68466596034eb883a93e67 : SUCCESS in 1h 13m 50s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/e9cb6a5fab574a73875723d6bcec3b02 : SUCCESS in 1h 13m 01s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/d7837177c5c14fb99f0a1daf102c9722 : SUCCESS in 48m 46s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/6aaf7306bfbb494b95787536d6741d1e : SUCCESS in 1h 18m 12s (non-voting)","accounts_in_message":[],"_revision_number":8},{"id":"351c9bc4c5ac5de918fff97515061748606018b5","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-26 21:11:29.000000000","message":"Uploaded patch set 9.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":9},{"id":"da79bd98983cf5be3b6b9caec8413a1f6efcf250","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-26 23:03:44.000000000","message":"Patch Set 9: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/7ef9f706b8f6488b978eb04cc05ce669\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/c8102704509b4158b1e2860b326a3e75 : SUCCESS in 1m 49s\n- requirements-check https://zuul.opendev.org/t/openstack/build/a5982190bd0947678789f550487c13d6 : SUCCESS in 1m 47s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/455a26c80cd748578ba39ecdd4006a26 : SUCCESS in 1h 45m 41s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/ffb80953c75f4239835b7c2406baf181 : SUCCESS in 5m 10s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d8fb3124f85c4aa2a968b4855a98995e : SUCCESS in 3m 40s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/4ba03a3ccf72423099faca6f0c6549fd : SUCCESS in 4m 37s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/670f40ba8f8a42cfa8e62d667153582e : SUCCESS in 4m 00s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/541ccdf345624ed38679350afcf6c791 : SUCCESS in 7m 57s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/c6780204041f4f318c12ec089704dc82 : FAILURE in 6m 12s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c78ae25c2fbe478b8d8cd4d634bcb1ca : SUCCESS in 3m 33s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/f808b9f020a848ec80bd7f3b403314e1 : SUCCESS in 2m 25s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/1f6cdf8142124cf49b957108a425927a : SUCCESS in 5m 00s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/0b947f01cf094621a75b0f3bf3d9cca4 : FAILURE in 3m 16s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/4a57d0a307f14e0ba69ddef17097e609 : FAILURE in 2m 36s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/d470683d1cee4adf9c628c3e74009d65 : SUCCESS in 1h 43m 38s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/6a935b6c0dbc486a8b08392c9356b665 : SUCCESS in 1h 19m 26s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/2da41235bfa04ce9b12b68b0b43ff9ae : SUCCESS in 1h 07m 52s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/90bbd739d163497495ee772665d1e2b4 : SUCCESS in 38m 30s (non-voting)","accounts_in_message":[],"_revision_number":9},{"id":"ea4313cf64a0f117dcde768aca7100dcf841a566","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-26 23:22:34.000000000","message":"Uploaded patch set 10.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":10},{"id":"386ddc7542baff068ad2c2b2bcc792d0a93d395f","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-27 01:17:36.000000000","message":"Patch Set 10: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/f5a80acc37974ffda63d789c3771d99f\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/76cdfbb02b7c4981a3087ae24aec10b2 : SUCCESS in 6m 27s\n- requirements-check https://zuul.opendev.org/t/openstack/build/9b855078ef7c428aab9d9e5191a088fa : SUCCESS in 1m 41s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/b472c702340d421c83f85b8edb84b694 : SUCCESS in 45m 53s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/9e89c751558f4c84a08339166178431b : SUCCESS in 4m 20s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a9da2b61a1f6459eb3323cd2ae02be46 : SUCCESS in 3m 33s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/b073e422cd0741d784fa949050e89aa8 : SUCCESS in 4m 06s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/56f38286de164d499b96679f08fc10d9 : SUCCESS in 3m 14s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/a2085876e75f4774b62fb74b0b9e2ddb : SUCCESS in 5m 57s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/c5f118523b634cf9bd6b3cd2ebba7b57 : FAILURE in 4m 26s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/563ffc06650d42f28cd67ec8974867c2 : SUCCESS in 3m 24s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cce28cf37a6249788450aa47e23b8feb : SUCCESS in 3m 50s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/cc2e7c4765284e8e8309441ea07216f2 : SUCCESS in 5m 19s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/7408073227034a24923fa83be1395f39 : FAILURE in 5m 08s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/c35228fab3b440a3a5919f4df81a4c45 : FAILURE in 3m 17s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/e64aed77e346443391fd7a2277680ac5 : SUCCESS in 1h 09m 28s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/1e16b7de8832462b94a31ed6ff9fbe1f : SUCCESS in 1h 47m 48s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/ce658ef0108b4653bbf9862ed8bdb7d0 : SUCCESS in 1h 04m 03s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/7f753af2f70b4cee820eaf569f3038a6 : SUCCESS in 1h 11m 30s (non-voting)","accounts_in_message":[],"_revision_number":10},{"id":"dce7341cc27eceb733d91ff2edd80410fdb11ec6","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-06-30 06:48:10.000000000","message":"Uploaded patch set 11.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":11},{"id":"280183bdfb000594ff79176a144450f3c4efe065","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-01 07:00:07.000000000","message":"Uploaded patch set 12: Patch Set 11 was rebased.","accounts_in_message":[],"_revision_number":12},{"id":"16a6bc21f62943bcc9d340cec7e7360f96ed180f","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-01 08:59:10.000000000","message":"Patch Set 12: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/b17b99307b7c4fc1899193d8742492ac\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/5b2c3060f51b45a49460411ce29e645b : SUCCESS in 2m 12s\n- requirements-check https://zuul.opendev.org/t/openstack/build/829a0e20a17547b4815ee224ef27fe73 : SUCCESS in 2m 27s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/a2a486e72bb34f349f821cf6569a11c0 : SUCCESS in 42m 08s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/485c9ea2373f40eebff0794f817cac5e : SUCCESS in 5m 11s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2b7fe3794ba645d3a2023d628a0b4b1b : FAILURE in 2m 15s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/77936d7c4dc34345aa5f9d905d090215 : SUCCESS in 4m 11s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/684996b8a190411390615cd3ab808b0c : SUCCESS in 3m 30s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/aad5a69d55944f85b28be03e00f2bcf1 : SUCCESS in 6m 40s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/2cc2da78e089439faab2434a5ccd5ecc : FAILURE in 8m 28s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/99d260a9e354428799abec74d73f35fe : SUCCESS in 3m 14s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/55b82b1faa504d0d98fdbd4f74cc9eaf : SUCCESS in 2m 33s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/c018da1780494bae84dddf8e8cb98984 : SUCCESS in 5m 27s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/597e9f6b80b14ab0919e04fc295f2380 : FAILURE in 9m 22s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/9b26edbfd1bc46f186c332c996adc12c : FAILURE in 4m 14s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/ffdd71a6629445f2840e9922c60cbe3e : SUCCESS in 1h 01m 31s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/5334350138da417ab10096ad4f30b91c : SUCCESS in 1h 52m 03s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/d50e8ec49cc2436bba523f495717b2be : SUCCESS in 1h 03m 12s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/ca92b96734a64d0a858840fd7ecca113 : SUCCESS in 39m 48s (non-voting)","accounts_in_message":[],"_revision_number":12},{"id":"09cb95672e46c48966f743877d5a1b4356ced826","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-08 05:39:03.000000000","message":"Uploaded patch set 13.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":13},{"id":"9e421c349bf24cb7e26f693cf9d92985b5c44d0c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-08 06:19:36.000000000","message":"Uploaded patch set 14.","accounts_in_message":[],"_revision_number":14},{"id":"2dcc747c52f809f2ee80155eaef8a858bfb0bb0f","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-08 08:31:15.000000000","message":"Patch Set 14: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/7c546e513a87498ca2934be63ae14a6c\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/0875c35f730c45a4ac9d3a8d7faa28d2 : SUCCESS in 4m 05s\n- requirements-check https://zuul.opendev.org/t/openstack/build/fc5abcaa9cf74d98af4f630809341a0c : SUCCESS in 5m 40s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/9dd757a5cae0460786e2d0d3f0615b55 : SUCCESS in 1h 34m 55s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/646ad943431b4ef1819f765607f3a958 : SUCCESS in 4m 58s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2725e2cfaf6c4d83b33ba89659cc8247 : SUCCESS in 3m 25s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/6f86f398adc74b348fd9987ec0beb16a : SUCCESS in 4m 18s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/668cb93565d14d90956c72469a13fa5b : SUCCESS in 7m 06s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/aa1004296566496fb6cb563ac5b06ad6 : SUCCESS in 10m 47s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/ee0bb29d7cf94934aa478f3aff8e2955 : FAILURE in 6m 08s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/47fc9ddbcff24a70b89367f2d1357107 : SUCCESS in 2m 25s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/91d6675207904a87873c59bed1d71a1f : SUCCESS in 4m 55s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/b1208a6dcbc94f1bbc4e9b5ee4061d1c : SUCCESS in 5m 03s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/1ef63eef0d114902a37544170b448fca : FAILURE in 3m 36s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/e61b7dd9ddc74f9db8829221248acae0 : FAILURE in 2m 37s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/cd1effd7af8f4b1a9da186098eb379b7 : SUCCESS in 2h 03m 29s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/d538d1a4eb124edcbbe541fd466d1da0 : SUCCESS in 1h 54m 40s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/39be723719f54fbf89498fc92d63c300 : SUCCESS in 1h 04m 26s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/edab3a0112d44066a31fae4c04f88591 : SUCCESS in 1h 20m 19s (non-voting)","accounts_in_message":[],"_revision_number":14},{"id":"8d09370380bbad569daa81fbda535e2fd786e1d8","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-08 09:17:33.000000000","message":"Uploaded patch set 15.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":15},{"id":"c8abfd3fd353a345340dcb90d3ee05ebd5493e69","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-08 11:29:48.000000000","message":"Patch Set 15: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/39f083891b2942f4a9fad8403d718b94\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/e4c7af2172424060bdee8ea6fa920144 : SUCCESS in 3m 14s\n- requirements-check https://zuul.opendev.org/t/openstack/build/f361c3b54003403d9574b87295fc3add : SUCCESS in 2m 33s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/c8d616da6acc49ce91f2c4070977274d : SUCCESS in 1h 32m 18s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/36e693ce9110431aa5400e012df036bd : SUCCESS in 5m 03s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/9391112462ca41cfbddb36a70a8183e4 : SUCCESS in 3m 15s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/b43559c8c90448c8a4e58e2862c8b5fc : SUCCESS in 4m 44s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/1f87d027b70147c7bf595b53fdb496c9 : SUCCESS in 4m 31s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/817fcbbfcf6447e3b4f62b7da3c1f273 : SUCCESS in 7m 06s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/0c5de772a906412da4c1dce2673d4495 : FAILURE in 3m 54s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3e7147080f1d464096b6f8761d0c91b8 : SUCCESS in 3m 40s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/7cd964af61e04048a0b8fbee6b2171ef : SUCCESS in 4m 34s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/75d972bb5d294059a85589b5723f7b4c : SUCCESS in 5m 24s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/14a35d7cc5af45d8938e48ef93714c21 : FAILURE in 8m 04s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/3bd4244d89da48d5832b8cbbd51bf02f : FAILURE in 2m 43s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/1efd5e9c8b8042d48b1d26f4eb4555ba : SUCCESS in 1h 41m 45s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/dc45c65d83fd4e7380ed4b99d16b3836 : SUCCESS in 2h 00m 13s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/ed703496c4524aac824129bb65311746 : SUCCESS in 1h 06m 24s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/e183f72c09b941d89103277c8bd7eba8 : SUCCESS in 1h 17m 16s (non-voting)","accounts_in_message":[],"_revision_number":15},{"id":"c48de51b9ff9570a3a3fc24bacf18426d92eaf7e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-08 18:29:40.000000000","message":"Uploaded patch set 16.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":16},{"id":"4bdd432635aa0447c6556fdc5b99999872c0e5b5","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-08 18:52:48.000000000","message":"Uploaded patch set 17: Patch Set 16 was rebased.","accounts_in_message":[],"_revision_number":17},{"id":"32240821e786f5603b3dafb17ed27e4ee34e82e8","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-08 20:44:46.000000000","message":"Patch Set 17: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/387aeb492828475faa6f8d8c88fa991d\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/4f96ac76e18d4c85b47d32d5fe594081 : SUCCESS in 3m 09s\n- requirements-check https://zuul.opendev.org/t/openstack/build/31862895d61a42f4b2e46788a1c5c7b9 : SUCCESS in 2m 37s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/dbce8a4714464613b83721ed709ad2fd : SUCCESS in 1h 32m 40s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/19b7da17133a4beea3d956dbcd5a7c4b : SUCCESS in 2m 58s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/98f9730e3de5489a904e02186beb9c97 : SUCCESS in 3m 35s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/c42bbca3e4854a34b74dcebb52c18d75 : SUCCESS in 4m 20s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/5be27905a5084696a105ae9d554ca7ee : SUCCESS in 3m 54s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/5df2fcc3133049b6b28b17ea3d7da916 : SUCCESS in 4m 22s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/b3f7f64d6fa048429969884cdd30b26f : FAILURE in 6m 00s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f0df22e013ed4566bbb79761de8b5f72 : SUCCESS in 5m 04s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/96b912521f36428bac4d9a4b541ac350 : SUCCESS in 5m 25s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/dc0b24030d134aa980a57e858554336c : SUCCESS in 5m 22s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/1e152185bcc6466ba67833a5e1fa65d8 : FAILURE in 6m 38s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/0fa9d6fc47e54ef5a2b058b0d3e8272c : FAILURE in 2m 58s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/6373cca2f5694b4784acc131c40ea325 : SUCCESS in 1h 13m 20s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/210640c99c86402f9739bd00f666e8d0 : SUCCESS in 1h 47m 10s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/51ec9838ab2b47e2b81cdc8f134cf202 : SUCCESS in 1h 15m 06s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/62a33c007f6f4b88ac9b8900630df21a : SUCCESS in 1h 27m 51s (non-voting)","accounts_in_message":[],"_revision_number":17},{"id":"865a650df8660924c9ee723a00b4146235ba1fd7","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-09 05:22:05.000000000","message":"Uploaded patch set 18.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":18},{"id":"97b6504bdce10061b7e81e5179c991750ebdddc0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-09 07:01:45.000000000","message":"Patch Set 18: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/5413b53af874481ca848f48a2897e9dd\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/5b701a19800b46bd8f018b015562c7e4 : SUCCESS in 2m 44s\n- requirements-check https://zuul.opendev.org/t/openstack/build/511a7aa73ed64f97b5c3e751c3d2b2ad : SUCCESS in 2m 21s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/d141efeec3f7483f874d50f48c2dabbe : SUCCESS in 1h 35m 14s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/be551763afaf4f11b5145c5e0e5d272c : SUCCESS in 2m 56s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/027a2447b2864261a615fbc6a30ed03a : SUCCESS in 2m 39s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/d80f1557eaf34da39dd55cf027444c90 : SUCCESS in 2m 47s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/86c77fbe46ca40f4997f6b7369d9f942 : SUCCESS in 4m 28s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/3527c82d33294dea974d11da707114f3 : SUCCESS in 8m 08s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/41d8df0ebcfb45549e4cbe4cc333fc8f : FAILURE in 7m 30s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/eea1df82f33641f0a02205eff77924d6 : SUCCESS in 2m 59s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/6d20326a4b1f4cf2adebcaf4394f9825 : SUCCESS in 5m 34s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/acb9621265ab4b5690fdfc9993eb30a4 : SUCCESS in 4m 52s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/a6843e199988441299c2617ecdcb3216 : FAILURE in 5m 08s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/ff21bf46266c46d0aab694a3d81924a0 : FAILURE in 2m 37s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/e23364d030114f5c9100c1a214e90c89 : SUCCESS in 1h 01m 11s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/406a7affb8104c09b8cb6728c93ee70b : SUCCESS in 1h 02m 18s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/5128db5a0fad471bb4830e38677da13e : SUCCESS in 50m 40s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/8486e2b816c64165b1f3639d640c451d : SUCCESS in 1h 18m 24s (non-voting)","accounts_in_message":[],"_revision_number":18},{"id":"c1cad837d2d32f1360c5351a892b5fdde0506323","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-09 10:18:00.000000000","message":"Uploaded patch set 19.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":19},{"id":"5b96fb765427eb438cbc8c8f0abcf60af98410d7","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-09 10:47:31.000000000","message":"Uploaded patch set 20.","accounts_in_message":[],"_revision_number":20},{"id":"8d7926c692a68e499fc2e1db33d2a79fed8a10fa","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-09 12:42:53.000000000","message":"Patch Set 20: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/aa2652736c9549e9bfb10544761f5c17\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/6a198b79314b455389dde2a48c1e0d18 : SUCCESS in 3m 28s\n- requirements-check https://zuul.opendev.org/t/openstack/build/e5ca75c5d61449eab874c9be1f6b2179 : SUCCESS in 2m 47s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/04a574dfc1724f39ab0a563039e1e61e : SUCCESS in 1h 37m 44s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/9e26453bec51473895c47670700c4ca7 : SUCCESS in 4m 16s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/568ed54fc4e64667b75811f3e250c30c : SUCCESS in 3m 25s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/f3a0cdd2445645b8bd16eb13062e93cd : SUCCESS in 4m 18s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/577400d638c142839ebe52f8f59bfc75 : SUCCESS in 4m 23s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/bd48776e946648a89d09c1e9f67e05a6 : SUCCESS in 8m 40s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/6a5f238f4c9b4f57819ae2b5d3084323 : FAILURE in 5m 47s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/29a89112fe4440b2b8017c230565c759 : SUCCESS in 7m 07s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/ffe9496a2b244ebfa889ff83573dd2a9 : SUCCESS in 2m 28s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/4f49ed18ba29494184936e65ee0a9bff : SUCCESS in 5m 11s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/aca294cbea464d72ab98a8ec1df1b05e : FAILURE in 5m 46s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/7bc083fd2fba42598c03cc26f43847c7 : FAILURE in 2m 00s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/e052b35689ad4699b8114388a5d2ede6 : SUCCESS in 1h 47m 05s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/f4021fa874fd466996c81fefca62d936 : SUCCESS in 1h 39m 36s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/20f7842d2050466d877e73a57a272775 : SUCCESS in 29m 03s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/e18a04a8eb4c4455a9f8325bd526a877 : SUCCESS in 57m 23s (non-voting)","accounts_in_message":[],"_revision_number":20},{"id":"a2997bb0e09e2ff612754648e105f74e4bba75ad","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-09 23:23:38.000000000","message":"Uploaded patch set 21.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":21},{"id":"2cd6b99119cc4d37e8fcfff59af105d8194bd0c6","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-10 01:09:41.000000000","message":"Patch Set 21: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/4ce1c0bbb5bb42c2acadc9d77ea7c6cd\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/fc7945f1512a4df79653e6c5da8d6c99 : SUCCESS in 2m 38s\n- requirements-check https://zuul.opendev.org/t/openstack/build/478e9aa02aa44807a0fb50181975c7d0 : SUCCESS in 2m 00s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/f82567759d5a4b52abc2a72cb13752ee : SUCCESS in 1h 39m 07s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/ddde7e980ac9466b9ab974a32f4176cd : SUCCESS in 4m 33s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/6ef6ab2961bd4b758fe0634abab4e4fd : SUCCESS in 3m 16s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/393b51939a1f453fbf009012dadb1424 : SUCCESS in 3m 31s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/97cd088d6c53418887d55911f8c53d57 : SUCCESS in 3m 43s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/3edb421a9fca457e922faf965530ca54 : SUCCESS in 6m 14s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/5f14e4d19cbe4406b24d3fbe82c9c0a6 : FAILURE in 5m 13s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/2a1d1c0705dc4dc281a9e56ace7c13e0 : SUCCESS in 5m 02s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/81012e66d78a49a5a55b2f3f2b585c50 : SUCCESS in 4m 20s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/84461dc40b334b0c98c1ec64685dcb21 : SUCCESS in 5m 34s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/c7664dac33c94603ad9d581ecb91797b : FAILURE in 4m 24s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/67151acb1af1464e810b9df5c8f82c6b : FAILURE in 2m 58s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/c0ecfb9201894eed9a3bd8254e45bca3 : SUCCESS in 1h 07m 00s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/393cd2287aeb4b6482fcdb2cfa4b49b9 : SUCCESS in 1h 11m 31s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/7a8aba47b6b74305bcb549d35f51323f : SUCCESS in 1h 04m 02s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/c15b9bfa1b5c4f4a8c5d519c2bc27c9f : SUCCESS in 1h 16m 40s (non-voting)","accounts_in_message":[],"_revision_number":21},{"id":"f3d00bdd513f97440e3ec9a68ae49fd9e54a8f40","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 02:30:33.000000000","message":"Uploaded patch set 22.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":22},{"id":"f3ef0f2200e94db56d0012467a591d85f1223b2d","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-10 04:37:40.000000000","message":"Patch Set 22: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/bac589047bfa42d3b72c84428c923920\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/d11d37c378d04179b7fdefe20920de51 : SUCCESS in 5m 04s\n- requirements-check https://zuul.opendev.org/t/openstack/build/0a89f030021146cfad398001b6b32eff : SUCCESS in 2m 51s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/2598c742b41f40e58018fd53f16d1836 : SUCCESS in 1h 33m 48s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/fd66b3efb7e4405ba31f5528cf1f7e9d : SUCCESS in 7m 36s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/8777b644fae147798dcd539d65797602 : SUCCESS in 5m 19s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/f3ac6d2402774eeda9806487104cf02e : SUCCESS in 6m 09s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/8fe7ef89a4e1465b8bc5a63139edbf51 : FAILURE in 5m 17s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/38875636e44d4c57a2b098d9c12ed2a2 : SUCCESS in 6m 45s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/767243ed9c4d491480693639ecf95532 : FAILURE in 6m 04s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4ebc8c27d60248e2ab97abb2df137f9b : SUCCESS in 4m 43s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cf0a4b7ad2c0428c9598c2e05e8a8439 : SUCCESS in 4m 29s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/d520b82d210844a2ad0c66d868b71c79 : SUCCESS in 7m 18s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/af0a4948faf3494ca8c5fae546cce677 : FAILURE in 3m 59s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/6e14710d4e374094bbfd489725a291fd : FAILURE in 3m 17s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/751e69f80ab345a6a75dc7fe2e8b2b53 : SUCCESS in 1h 03m 45s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/038e29e61fb94a30b9b204d93b4ca7b2 : SUCCESS in 1h 14m 09s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/91cbf7741b3d413081558aae019adb14 : SUCCESS in 1h 12m 51s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/950e718d3cef46bda58bad8a50267c67 : SUCCESS in 1h 05m 12s (non-voting)","accounts_in_message":[],"_revision_number":22},{"id":"fa6805d19390017059bf529abe2ea42cc5d37994","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 04:51:06.000000000","message":"Uploaded patch set 23.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":23},{"id":"cec0efa2373419b21eea938002ceecdaccbc1e71","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-10 06:15:16.000000000","message":"Patch Set 23: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/5eae191886bb46a0a393906ec272d961\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/0b8e0c98c2ea474da20f1fc95d8df758 : SUCCESS in 2m 14s\n- requirements-check https://zuul.opendev.org/t/openstack/build/df4c203fad9b468a8df4e55b6e7c838f : SUCCESS in 1m 46s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/395221925c134ebfbd1335dfc6e56068 : SUCCESS in 47m 35s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/b80146eccf5344a1ab696fb899dd4193 : SUCCESS in 3m 00s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/95c7aead12ce42a08e9d69e686303d1c : SUCCESS in 3m 23s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/039b30cd92734352813266d1fbabb7d1 : SUCCESS in 3m 57s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/746ce35bab0948dfa126bfa306d7e4b1 : SUCCESS in 2m 37s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/dfff885afe4c434fbfda0c94c45981f1 : SUCCESS in 4m 51s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/7ca9a5050dcd43fabffc5ba8fba8836e : FAILURE in 6m 05s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/37a79766878a4729bbafa9a984ea2f0b : SUCCESS in 3m 44s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/c5632fd96325458b8ed7f782e8eff83f : SUCCESS in 4m 17s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/0cb7cd8d884e44fe9d2ccfa4f2dbbad4 : SUCCESS in 5m 19s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/88f2a41f67c54fa0ba90c24caf9799eb : FAILURE in 3m 15s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/2c89a159c2a6464bb58a5fbdafb6551a : FAILURE in 2m 06s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/05247e54ac6a45deb1d4de75ebf09065 : SUCCESS in 1h 08m 40s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/3425bc2833fe4c0b82c25442c6a8a4d3 : SUCCESS in 1h 14m 32s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/6027a372d4a64e3fb39afc57e6ff6760 : SUCCESS in 36m 23s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/ac85b3afbdd748daaa8f718144fd2a20 : SUCCESS in 1h 17m 03s (non-voting)","accounts_in_message":[],"_revision_number":23},{"id":"2d899e7f00d210b9ceff5eb16c65af5604c5269e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 17:57:46.000000000","message":"Uploaded patch set 24.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":24},{"id":"ba7b1719ca83c2d8e3f351cf49bb0b38086055f1","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 18:13:55.000000000","message":"Uploaded patch set 25.","accounts_in_message":[],"_revision_number":25},{"id":"755830647c815ee1f7ca038da3da78976416978b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-10 20:13:59.000000000","message":"Patch Set 25: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/0af0328d8139484b9c8b6ee3e2b0ea52\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/a673c0e6e7cd4ef2b9f3268d82779db6 : SUCCESS in 4m 07s\n- requirements-check https://zuul.opendev.org/t/openstack/build/ee12a34352e54c4ea3314a2cf323f1c2 : SUCCESS in 2m 28s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/2d4e86c3b09441af9690d2b9e8933313 : SUCCESS in 46m 43s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/d826aa2930754d3e8f07b1ff904bedbc : SUCCESS in 6m 01s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/449a949cc692401fbb26cd68e06efc6a : SUCCESS in 3m 07s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/4544d8cd5d83499480597fcccf849350 : SUCCESS in 3m 56s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/14330cbf205e446d9e323b9b9c7e42b4 : SUCCESS in 3m 47s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/b426954f5a584fc5b480ec3b5c2f62c8 : SUCCESS in 5m 42s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/09694bcd601745ec94823da2943a1ad2 : FAILURE in 7m 16s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b6f501e2f0654925906fbf1754150ad1 : SUCCESS in 3m 36s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/daf5becc96f149fb8d78d9ba80f71e42 : SUCCESS in 4m 45s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/5c09e5192fef4763ae9b50edf6656990 : SUCCESS in 5m 37s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/6bee49a1b26e45bd88566ab577d5492a : FAILURE in 3m 14s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/19089cf65c7b477f8a715fc6aeb3713d : FAILURE in 2m 32s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/a4a5d97a2f874326b9d9d95b288123b6 : SUCCESS in 1h 52m 42s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/30662fc08814436796f8dadb02158a8a : SUCCESS in 1h 12m 14s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/fcf8f5a995e94ff0a7a320d1a648ad7e : SUCCESS in 35m 52s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/88fa6f0ca3d14b27b8bc68002a939eef : SUCCESS in 41m 45s (non-voting)","accounts_in_message":[],"_revision_number":25},{"id":"a04cc848ef4b6933af9d0f702c1412e1574f41fb","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 20:28:49.000000000","message":"Uploaded patch set 26.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":26},{"id":"ec4ef9887162950944b124304d7b70e68b71cf16","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-10 22:12:50.000000000","message":"Patch Set 26: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/7061bf6178704f61b1709294437a29c1\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/ddcefe7372364a03b728e8596e100385 : SUCCESS in 2m 34s\n- requirements-check https://zuul.opendev.org/t/openstack/build/82c60cc43c2a4e5397756022dab0a360 : SUCCESS in 2m 12s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/404db8e8e63146aa95845f5fd15b9efe : SUCCESS in 1h 18m 32s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/11f6887e16064ce7abafce9892082f76 : SUCCESS in 4m 59s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/84960ea0993443f3b3a1eed61c829b10 : SUCCESS in 4m 38s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/ae59cecff13c461c95b0d43f79d6fe73 : SUCCESS in 4m 14s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/49241d75985648519c2e11095b5289ea : SUCCESS in 3m 07s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/fc4c3c325bdf4ca69149a18a6054c531 : SUCCESS in 6m 30s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/fffcd0da83cb4af4b4e9bc2ddb1ebfdd : FAILURE in 6m 16s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/dbab8318cf8a4d68aaccbbae18c5f34e : SUCCESS in 2m 43s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/b14692401ec749508d3c96b096385a8a : SUCCESS in 2m 55s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/061f1d8dbd5e44d1931ec4f6a34bfaf9 : SUCCESS in 4m 30s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/02d6da6b9a714670821d05b5b56176ae : FAILURE in 4m 43s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/3d03dd7eb5c04deebe92c2b0be3043f5 : FAILURE in 2m 16s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/022b0ebe2f794619b00ad4d52e1a73fa : SUCCESS in 1h 00m 35s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/e5fd1b4764cc4626928e3c9e8ae7ce08 : SUCCESS in 1h 36m 38s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/d375398e6d75441e91c139f63d016304 : SUCCESS in 33m 52s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/063b67f0c89d4784b70576d0b39129b6 : SUCCESS in 1h 15m 31s (non-voting)","accounts_in_message":[],"_revision_number":26},{"id":"de67d12bf5048764dcd821fce939108d0607f0c7","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-10 23:06:58.000000000","message":"Uploaded patch set 27.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":27},{"id":"0eb3874f91819e9d56ed9a5189fbcdbf02271fd0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-11 00:52:45.000000000","message":"Patch Set 27: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/08109d249b01459b932027b143dfb47a\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/805262efa15c42158e093504e58dd3a6 : SUCCESS in 2m 52s\n- requirements-check https://zuul.opendev.org/t/openstack/build/672ce25aff5248239473b087c1103ae7 : SUCCESS in 2m 10s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/e20c85f4abc9483fbce59d3941965dcb : SUCCESS in 1h 37m 50s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/8ebae0d575ce4ad8ae9d353a9bdf976f : SUCCESS in 8m 09s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/23fc8bde883a473baccde495d5f63760 : SUCCESS in 3m 02s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/0c7bee0e57d949a49d8a47323aaafb12 : SUCCESS in 5m 12s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/63f977008b044415bd292c3326b2490f : SUCCESS in 5m 23s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/f9b8e3e160944e26aafc498a87161c86 : SUCCESS in 6m 38s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/84543f21794348b683096db206d9104f : FAILURE in 4m 59s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4f0a1c8a437a411284c2790404ff059f : SUCCESS in 3m 49s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/630cff98b3744607ae81fc0477d2534a : SUCCESS in 4m 22s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/8f2e94976ead47d5a4e2b14157628a11 : SUCCESS in 5m 11s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/f2bf955056f4449585b82ab6d345147b : FAILURE in 3m 58s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/787bb94229c344c7aadf3cfbdd25cb1c : FAILURE in 2m 23s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/51ee142f206d4ad1a30c7bdd4bdcaf7c : SUCCESS in 1h 25m 19s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/83ab58a6994d4ae59d2831f2d460bc50 : SUCCESS in 1h 28m 59s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/612f2f011d7047c0a3c880384cbe49be : SUCCESS in 1h 02m 54s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/76edd444ee20466faf6e37d5ae3140e5 : SUCCESS in 54m 46s (non-voting)","accounts_in_message":[],"_revision_number":27},{"id":"654d5fc6f3bf149f71e1735f5a41973d27627225","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 01:26:56.000000000","message":"Uploaded patch set 28.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":28},{"id":"5c6d9579df0bed5eb4fb103e81c079ffabcc7ec0","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 02:35:08.000000000","message":"Uploaded patch set 29.","accounts_in_message":[],"_revision_number":29},{"id":"1e4943bb17bbd344ce08f91448dd01c4fafef514","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-11 04:45:41.000000000","message":"Patch Set 29: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/654c1e0f5e7e4a4db91430e16fb1bfe6\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/4f2c431fdcab4465871d25ee3b668e7b : SUCCESS in 4m 30s\n- requirements-check https://zuul.opendev.org/t/openstack/build/c999620dd10442be8c7e5283c7ffadb7 : SUCCESS in 3m 00s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/6e7902693c06487d8a1314926978e34b : SUCCESS in 1h 34m 57s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/396ded5f9f064a7ab53a26449332c926 : SUCCESS in 6m 29s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/e396e32e75334f08a85afffb1826f6e7 : SUCCESS in 3m 15s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/6610ab70f99d4aa08cfae6abd15cf1ee : SUCCESS in 5m 54s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/c606e36554ae4835a3977070a9157a4d : SUCCESS in 3m 16s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/957644e415134e0bb9cdabb2e790267d : SUCCESS in 9m 17s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/519ea39866324419ab007f50a72f15cc : FAILURE in 7m 56s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/cc9b85f86cbc4ea99f514c313a6a25ea : SUCCESS in 6m 03s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/75ed613b29bd4bec8aeff8172eb20091 : SUCCESS in 5m 28s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/3f43beb42e8542b1901eb59f3aa53019 : SUCCESS in 7m 44s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/a56e7c9a9a124be59dc4a2142698b3b2 : FAILURE in 6m 20s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/b3e09eb1079f45cd8fbe1588524d9f85 : FAILURE in 3m 10s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/c8d0ba438b8f4fb78ff9e4f22ebd7b32 : SUCCESS in 1h 27m 40s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/34ce0c5a4e0240ac94e45d1dadc278e4 : SUCCESS in 1h 37m 13s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/2c4c9c01e0aa40a8a483fb90dd352014 : SUCCESS in 50m 56s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/9fe908158a9041c79a4d45388cc4feb3 : SUCCESS in 43m 51s (non-voting)","accounts_in_message":[],"_revision_number":29},{"id":"20405b25c2181eefdcc062b8ddd0c4a04ec7d2c8","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 08:11:42.000000000","message":"Uploaded patch set 30.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":30},{"id":"e34807cd3058797f518ea280d65e3907baa35313","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-11 10:03:33.000000000","message":"Patch Set 30: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/dbcef6038a534e769280b7042155e670\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/36df351055984ac79f7e62d84efb6928 : SUCCESS in 3m 11s\n- requirements-check https://zuul.opendev.org/t/openstack/build/4f6fbd26605f4a068244fad9a2af5d94 : SUCCESS in 3m 51s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/47caf3ae8f2e4049ab0e7256e0b5feb9 : SUCCESS in 1h 30m 06s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/b70b7d8d10ea4fe1adcb6cf5d4d784cc : SUCCESS in 4m 20s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/293a66657cc64db99ed8660988234986 : SUCCESS in 5m 15s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/6f4837abaef948beaaef4214f1b9b693 : SUCCESS in 3m 48s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/79acf61d9913463b90b9d120ada0053b : SUCCESS in 3m 55s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/fc5e958ec0d14944b83d10659b14bae9 : SUCCESS in 6m 28s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/e6bb875ce498463baa4f4c4a75b8a3c5 : FAILURE in 7m 57s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/cec0eb9c87db45969661a4e8bf6bd97b : SUCCESS in 4m 59s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cbc09735753a4632918b965b5d67d886 : SUCCESS in 5m 52s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/5b16eab6644d443b8428fb3321c68990 : SUCCESS in 5m 26s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/118f16193f8f4a009c2701c02c11c6ab : FAILURE in 5m 50s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/43ff1679a6f741688413e6e79d9cafbc : FAILURE in 2m 42s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/d332be80a0764c60965619c6aa06f2a4 : SUCCESS in 1h 43m 12s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/976a0afb240e4ac681527168abca3590 : SUCCESS in 1h 17m 18s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/8c987ffad1b54fd7b2d4382c99b779f8 : SUCCESS in 1h 03m 32s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/30b9d4dcfcf9437a97037bf90b68ccf8 : SUCCESS in 59m 55s (non-voting)","accounts_in_message":[],"_revision_number":30},{"id":"721a57bf0ce552af6d44334af763e035c4933601","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 20:02:12.000000000","message":"Uploaded patch set 31.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":31},{"id":"1b52e15e050a6fbf3e1d85c3e0f2de25b696b294","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 21:36:46.000000000","message":"Patch Set 32: Patch Set 31 was rebased","accounts_in_message":[],"_revision_number":32},{"id":"984722b36b157116658776e1773e79ddf275a1e7","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-11 23:16:00.000000000","message":"Patch Set 32: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/20497a792b5444efb01cfdf6505dee33\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/dbed0c28da7848108cb3460766958d6c : SUCCESS in 2m 44s\n- requirements-check https://zuul.opendev.org/t/openstack/build/d68861b664924eaeae24ec0acd5881f7 : SUCCESS in 2m 11s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/dea2a31b74cb4dd296d14a4c2ce9f2f2 : SUCCESS in 1h 20m 45s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/e300ddb003c9466aa72bcd82d0b3b2b0 : SUCCESS in 3m 17s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/5754e79e45814102bda01811154e41dc : SUCCESS in 3m 07s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/b854bdd4b83a4f5cb52aad54b41b0da5 : SUCCESS in 3m 15s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/c23556402b7d4d09bf59490c4e97b9ca : SUCCESS in 3m 50s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/0fb6215927ba4e12996c9556f6e82ea0 : SUCCESS in 6m 00s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/e641fc5690d44c37831787650075f4fa : FAILURE in 5m 30s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/72f09c15f1184224b40bd7873cbb3048 : SUCCESS in 3m 30s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/b735f3e67d524164ab5e8cec71c8ce74 : SUCCESS in 4m 07s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/0c98b234fdc543508f7bb3ef560dd32b : SUCCESS in 5m 14s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/88675ccd03ff4254bcce396edab009ee : FAILURE in 4m 27s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/76202dd8cb484b74bc9c7006f145aacc : FAILURE in 2m 44s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/9a593f0b45ec401ab1a25d6218797a61 : SUCCESS in 1h 31m 58s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/fc204bc013654c0ebf5e10b368cb1d94 : SUCCESS in 1h 08m 40s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/8b42d0371edb480da04b7ddb2b1376ff : SUCCESS in 1h 05m 47s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/7821d14af57843f5925d89b27ad64825 : SUCCESS in 40m 52s (non-voting)","accounts_in_message":[],"_revision_number":32},{"id":"5d4452a5cdfc0ce710e18995034746fb2029f2f8","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-11 23:31:49.000000000","message":"Uploaded patch set 33.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":33},{"id":"282517c2932d49fb76bc717c8904174923e34ade","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-12 01:25:13.000000000","message":"Patch Set 33: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/90a4f4a00325481398668f6445ae77e4\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/cb6e564dd50b4a8cb5a261984181cf1b : SUCCESS in 2m 36s\n- requirements-check https://zuul.opendev.org/t/openstack/build/81361252ed1a45759e2e3ab807ce1ae4 : SUCCESS in 3m 18s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/3d29e74f253d4bc3918716b31d64a445 : SUCCESS in 1h 30m 36s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/fc3f77907d9945289cefa128a0918e2c : SUCCESS in 4m 36s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/35e5f6e2547243f5a0fd160edcb78cf5 : SUCCESS in 3m 18s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/0e79dc2021504ba3aec3085fe1bbbece : SUCCESS in 3m 53s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/b829f2c4d6004c7b93fce0a50d94b784 : SUCCESS in 4m 09s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/fa6f670a748844a3aaeb2f3f5125b3bd : SUCCESS in 5m 50s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/21d33d877d6c4a41923c70aaa6b63bb3 : FAILURE in 5m 19s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/06121e7420654c34b345e89ba83bf588 : SUCCESS in 3m 35s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/2806fc2ded424ed0b57ccfb31e51a00c : SUCCESS in 3m 57s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/a3d5f1c62e8b4c509d6a39dc51d9b843 : SUCCESS in 6m 18s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/3b5da0d04f9a497986651bb1dabf3abc : FAILURE in 4m 23s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/44f1f11a44ed4a4d94c7700167336389 : FAILURE in 5m 44s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/6e9f0bb9f3d948568922aeaf3374843f : SUCCESS in 1h 45m 01s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/7cf5d2fb457b4c499c965f98c70161bc : SUCCESS in 1h 08m 39s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/9b85a68b00824173b0ad7bb318979eca : SUCCESS in 1h 03m 43s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/6fd86b7e9f8046e0898c69322c2aa91e : SUCCESS in 58m 17s (non-voting)","accounts_in_message":[],"_revision_number":33},{"id":"640df5000dca838e65e95134e026b996a0c0bb53","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-12 03:18:51.000000000","message":"Uploaded patch set 34.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":34},{"id":"3047588b398065e56c6f55397bf0ed415b1217fe","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-12 04:45:16.000000000","message":"Patch Set 34: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/b2b560a3128d468086bf8b021f7b1674\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/8dff18e25be045959c1a478b469fdd20 : SUCCESS in 2m 30s\n- requirements-check https://zuul.opendev.org/t/openstack/build/f78e546105244fb6a02b7efa5509512e : SUCCESS in 2m 28s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/512e7b1bf03c4381b266bfc49972a159 : SUCCESS in 53m 01s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/9fd3e888d0aa469cb05bd5ff1623c49c : SUCCESS in 4m 32s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/56bee398e2eb4c31b3a6674398dd0bac : SUCCESS in 2m 42s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/3abc28da35f44786abd1e4d7d074719f : FAILURE in 4m 47s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/7a6ceb15f8c7476e96e2b1fd04c87511 : SUCCESS in 4m 19s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/5f9d53216ff64f2d8fbc57fadbff287f : SUCCESS in 4m 08s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/d2f59ed74c404ebf8b1b3896ab0fa6fa : FAILURE in 4m 34s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/a9f968d1e0494e72aa91cd71c6c1495f : SUCCESS in 3m 46s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/f971a01d83b549178a416712ecbe7a72 : SUCCESS in 3m 04s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/e90045ffb97f4bf69269ad1f97cbaf04 : SUCCESS in 4m 29s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/99c8be5fe46b4366bc76fa1d3a25af84 : FAILURE in 4m 23s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/d498349067614fd9bb4f2ef25944817e : FAILURE in 2m 23s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/ce49e915fbac4d0683a5b2f8a9e68568 : SUCCESS in 57m 58s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/158b3aab961b4480b89362d4a69e6836 : SUCCESS in 1h 15m 51s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/4954b7dce65143bc850b446838f038c4 : SUCCESS in 1h 10m 00s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/19767a7a55fe462796ac213184a9591d : SUCCESS in 1h 18m 21s (non-voting)","accounts_in_message":[],"_revision_number":34},{"id":"cf973456d1e8a7365ce9c1af4ac40a70461dae58","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-12 05:48:53.000000000","message":"Uploaded patch set 35.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":35},{"id":"fbe0bca57cfbe03dcf78c2fe09a0bc00f78b92bc","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-12 07:37:54.000000000","message":"Patch Set 35: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/9b2d3318b74941e9b9d47a5f2e7d2e78\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/2c72fbc6029b4551960e8cb2011ff744 : SUCCESS in 2m 47s\n- requirements-check https://zuul.opendev.org/t/openstack/build/ed41731144554a9095ae1e9f1b8a2472 : SUCCESS in 2m 00s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/f94e1d39bc494ca098e67db552b92468 : SUCCESS in 1h 41m 26s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/0a9c9674d580458d98171024c236003a : SUCCESS in 6m 41s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2eceaaceaa0b4ceb8c94eaa4d833f558 : SUCCESS in 4m 23s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/e16b5844962f43549b82bd578b7a6a37 : SUCCESS in 3m 44s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/f2f26994588144e0a64298783c880e96 : SUCCESS in 5m 35s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/b68dc08dc28045219a0ff251d6e8df7e : SUCCESS in 4m 05s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/7f5f4181edb84928a65106ba8001ed28 : FAILURE in 3m 16s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/685af62f13a04d5bad801a858750e14c : SUCCESS in 3m 30s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/d5a30a58ae3b4e70b4d708173903c9b8 : SUCCESS in 2m 48s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/5297e6a75ac74a9fa11599527c1bb6ca : SUCCESS in 3m 56s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/699d9a4989cf41f68649f56a3bb15371 : FAILURE in 4m 38s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/121d78ab9bc842388c3e31f68e571235 : FAILURE in 2m 49s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/7e506c942ae1456ca43421bf97e50fd9 : SUCCESS in 1h 05m 52s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/8e587a91aed6467d9ffb174bc99a6eaa : SUCCESS in 1h 05m 22s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/480cd3439fbc4a9781464dc2d35c3093 : SUCCESS in 33m 54s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/1fa7206b0a2049788dcf9606fa7915a7 : SUCCESS in 1h 17m 59s (non-voting)","accounts_in_message":[],"_revision_number":35},{"id":"5e8f80b618f8ca4759ef6ae5257446122887523f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-12 09:23:40.000000000","message":"Uploaded patch set 36.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":36},{"id":"292297d9ed029ce3efa52a744a6e23ed267f4880","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-12 11:19:11.000000000","message":"Patch Set 36: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/934029a08b9241858a90e309ae4ea457\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/3eab6cab60bc4ad1a0af89bbfc8a4442 : SUCCESS in 2m 56s\n- requirements-check https://zuul.opendev.org/t/openstack/build/64d6c840d0d747d3818993a7a21cff68 : SUCCESS in 2m 29s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/c59fdd2b3f694dc38fd097b6a2adf814 : SUCCESS in 1h 41m 22s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/3c7f760bd87b4da59f43f0b54330bf89 : SUCCESS in 4m 17s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/045d5b590b6e4adda2b993c86c30d261 : SUCCESS in 3m 20s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/207f002ef3c54996b21e7241753e843f : SUCCESS in 3m 31s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/5666a2e3e4434104a8940205fbc837af : SUCCESS in 2m 54s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/809b8e203feb44c1885fadcb35540e6f : SUCCESS in 6m 16s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/b13f2b17da524c2790214dabf3d5ff86 : FAILURE in 3m 43s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f9ed4d5543a7477aa02cb431d42e1b48 : SUCCESS in 3m 18s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/7bf17752a0754fcdb1773fa7545897c2 : SUCCESS in 4m 31s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/0f4629393bfe411dad5dbd6dda722bbd : SUCCESS in 4m 35s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/4c85f1b738564af4a3835614aa46744c : FAILURE in 4m 16s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/8bc5d0123b474796a22369acd75291c8 : FAILURE in 2m 41s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/1e1b0c61258844d092e7d0d9e82d93bd : SUCCESS in 1h 35m 55s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/bf68c4abf54a404bbeb128903e046cac : SUCCESS in 1h 48m 06s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/a812bee07a964eb89533ea0d95d302dc : SUCCESS in 1h 02m 45s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/774c179aaf48455892304f7126eb50c0 : SUCCESS in 1h 22m 42s (non-voting)","accounts_in_message":[],"_revision_number":36},{"id":"a90c7f6e3405f83348ce04ce7ac9d4814d4f7633","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"date":"2026-07-17 05:41:12.000000000","message":"Uploaded patch set 37.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":37},{"id":"503dc6ab1b919165df6efa637a60e39fb163a95e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-07-17 07:23:03.000000000","message":"Patch Set 37: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/ee438a69218e43fb9ccf207eee808f22\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/8bb18902cc9e4ce08fbdeb49a5e24fbd : SUCCESS in 2m 18s\n- requirements-check https://zuul.opendev.org/t/openstack/build/953be445793d49ef81ede64ce1dc3a86 : SUCCESS in 2m 09s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/90851165a7904fa79aa58cdf17619e5a : SUCCESS in 1h 00m 32s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/927dd984735840f1bfdecbe895358c6e : SUCCESS in 3m 02s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d2e2de202bb84fb199e3b2b3ae4c90e7 : SUCCESS in 3m 14s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/245038ece6b44d5da2757559e8a7b83c : SUCCESS in 3m 13s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/1b0f3da1a4bc43a8862753c23347558d : SUCCESS in 4m 04s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/701882fb77604466b5cee690a422f01e : SUCCESS in 6m 23s\n- openstack-tox-py315 https://zuul.opendev.org/t/openstack/build/5cede6379c154753b0cf8eac5021a83d : FAILURE in 5m 13s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c25850196dd44e6b85e64603f079103a : SUCCESS in 2m 21s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/ba2b4e44da8847c1a80e6a62dd1c9d87 : SUCCESS in 3m 58s\n- oslo.messaging-tox-py311-func-scenario01 https://zuul.opendev.org/t/openstack/build/37cfd4a09c684f568a66432be4aa11bb : SUCCESS in 5m 20s\n- oslo.messaging-tox-py311-func-scenario02 https://zuul.opendev.org/t/openstack/build/71f42bdd081e4a978087736884fafc14 : FAILURE in 4m 34s (non-voting)\n- oslo.messaging-tox-py310-func-http https://zuul.opendev.org/t/openstack/build/b339a67ec75e40a4adb78ff47644f6c1 : FAILURE in 3m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://zuul.opendev.org/t/openstack/build/04d5ef5583474562afaa9c62459beda1 : SUCCESS in 1h 06m 08s\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://zuul.opendev.org/t/openstack/build/c0bc468a529d4ea2a610858b37b01633 : SUCCESS in 1h 37m 14s (non-voting)\n- oslo.messaging-grenade https://zuul.opendev.org/t/openstack/build/5f375e1f874f4790bd6cde798ee1a7be : SUCCESS in 33m 23s (non-voting)\n- oslo.messaging-grenade-multinode https://zuul.opendev.org/t/openstack/build/64f241d13f694229ac59e56a1592994d : SUCCESS in 1h 13m 10s (non-voting)","accounts_in_message":[],"_revision_number":37}],"current_revision_number":37,"current_revision":"4d8b9b5aa14cc027788efd5ab614358a6f95b559","revisions":{"b59bc60aed25da8721912e7533ae89c5bafa586f":{"kind":"REWORK","_number":1,"created":"2026-06-15 09:23:39.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/1"}}},"commit":{"parents":[{"commit":"c3a386396638447ae07b190ee5adad512b8c5f09","subject":"HTTP-RPC driver fixes (various)","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c3a386396638447ae07b190ee5adad512b8c5f09"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-15 09:22:57.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-15 09:22:57.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client, SNITransport, mTLS","message":"HTTP-RPC driver: httpx client, SNITransport, mTLS\n\nReplaces requests with httpx for persistent connection pooling and\nHTTP/2 support (via http2 config option). Adds SNITransport for TLS\nhostname verification when connecting by IP address, and mutual TLS\n(mTLS) client authentication (ssl_client_certfile/ssl_client_keyfile).\nServer-side mTLS verification via ssl_ca_file.\n\nAdds fanout retry with exponential backoff for transient connection\nerrors (SSL EOF, connection reset) via fanout_max_retries config.\n\nReplaces per-call Thread creation in RPCDispatcher with a shared\nThreadPoolExecutor to prevent memory leak from thread stack\naccumulation under load.\n\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nGenerated-By: claude-opus-4-6 (OpenCode)\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b59bc60aed25da8721912e7533ae89c5bafa586f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b59bc60aed25da8721912e7533ae89c5bafa586f"}]},"branch":"refs/heads/master"},"cd0c5c7a6c29d5eed41a9420b51693c2c61ece9d":{"kind":"REWORK","_number":2,"created":"2026-06-16 08:49:32.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/2"}}},"commit":{"parents":[{"commit":"c3a386396638447ae07b190ee5adad512b8c5f09","subject":"HTTP-RPC driver fixes (various)","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c3a386396638447ae07b190ee5adad512b8c5f09"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-15 09:22:57.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-16 08:27:05.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client, SNITransport, mTLS","message":"HTTP-RPC driver: httpx client, SNITransport, mTLS\n\nReplaces requests with httpx for persistent connection pooling and\nHTTP/2 support (via http2 config option). Adds SNITransport for TLS\nhostname verification when connecting by IP address, and mutual TLS\n(mTLS) client authentication (ssl_client_certfile/ssl_client_keyfile).\nServer-side mTLS verification via ssl_ca_file.\n\nAdds fanout retry with exponential backoff for transient connection\nerrors (SSL EOF, connection reset) via fanout_max_retries config.\n\nReplaces per-call Thread creation in RPCDispatcher with a shared\nThreadPoolExecutor to prevent memory leak from thread stack\naccumulation under load.\n\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nGenerated-By: claude-opus-4-6 (OpenCode)\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd0c5c7a6c29d5eed41a9420b51693c2c61ece9d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd0c5c7a6c29d5eed41a9420b51693c2c61ece9d"}]},"branch":"refs/heads/master"},"99a9cb15a5d4d55ddc2b17a6e3cb6e2a5e1c50f0":{"kind":"REWORK","_number":3,"created":"2026-06-17 13:29:16.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/3"}}},"commit":{"parents":[{"commit":"439084f1da55646801cc6df065edf533cc99f954","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/439084f1da55646801cc6df065edf533cc99f954"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-15 09:22:57.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-17 13:24:43.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client, SNITransport, mTLS","message":"HTTP-RPC driver: httpx client, SNITransport, mTLS\n\nReplaces requests with httpx for persistent connection pooling and\nHTTP/2 support (via http2 config option). Adds SNITransport for TLS\nhostname verification when connecting by IP address, and mutual TLS\n(mTLS) client authentication (ssl_client_certfile/ssl_client_keyfile).\nServer-side mTLS verification via ssl_ca_file.\n\nAdds the Service.tls_server_name field so the broker can advertise an\nSNI/certificate hostname distinct from the connection IP address, which\nSNITransport uses for TLS certificate verification.\n\nAdds fanout retry with exponential backoff for transient connection\nerrors (SSL EOF, connection reset) via fanout_max_retries config.\n\nOnly transport/connection failures are retried. A remote exception\nreturned by a successfully delivered RPC is wrapped (_RemoteCallError),\nexcluded from the retry loop, and unwrapped to re-raise the original\nexception to the caller, so genuine application errors are surfaced\nimmediately instead of being retried.\n\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nGenerated-By: claude-opus-4-6 (OpenCode)\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/99a9cb15a5d4d55ddc2b17a6e3cb6e2a5e1c50f0"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/99a9cb15a5d4d55ddc2b17a6e3cb6e2a5e1c50f0"}]},"branch":"refs/heads/master"},"93a8c394ec81a93eefd58ad5414e7296d641acc3":{"kind":"REWORK","_number":4,"created":"2026-06-23 02:12:14.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/4"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-23 00:41:42.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs via asyncio.to_thread; the reply/ack\n  events use the real (non-eventlet) threading primitives, and the\n  Hypercorn loop runs in a real OS thread for eventlet compatibility.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/93a8c394ec81a93eefd58ad5414e7296d641acc3"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/93a8c394ec81a93eefd58ad5414e7296d641acc3"}]},"branch":"refs/heads/master"},"a391c50575642c64c542c5e2f42ada2f3ecde9e9":{"kind":"REWORK","_number":5,"created":"2026-06-23 19:22:34.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/5"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-23 19:20:21.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs via asyncio.to_thread; the reply/ack\n  events use the real (non-eventlet) threading primitives, and the\n  Hypercorn loop runs in a real OS thread for eventlet compatibility.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a391c50575642c64c542c5e2f42ada2f3ecde9e9"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a391c50575642c64c542c5e2f42ada2f3ecde9e9"}]},"branch":"refs/heads/master"},"a6b655d66b783241fc269b5ea6e059f3bd3a869a":{"kind":"REWORK","_number":6,"created":"2026-06-25 09:09:08.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/6"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-25 08:19:59.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs via asyncio.to_thread; the reply/ack\n  events use the real (non-eventlet) threading primitives, and the\n  Hypercorn loop runs in a real OS thread for eventlet compatibility.\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a6b655d66b783241fc269b5ea6e059f3bd3a869a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a6b655d66b783241fc269b5ea6e059f3bd3a869a"}]},"branch":"refs/heads/master"},"a764b4f1fad2bd363e17297dc96f05ccbfd1e92d":{"kind":"REWORK","_number":7,"created":"2026-06-26 00:41:12.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/7"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-26 00:33:11.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs via asyncio.to_thread; the reply/ack\n  events use the real (non-eventlet) threading primitives, and the\n  Hypercorn loop runs in a real OS thread for eventlet compatibility.\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched: the ASGI handler runs in a real OS thread, but\n  the RPC server\u0027s eventlet executor must only be poked from the hub.\n  Calling the server callback directly from the listener thread woke a\n  parked green worker via a cross-thread greenlet switch and raised\n  greenlet.error (observed as a cinder-scheduler ack timeout that\n  crash-looped cinder-volume). Incoming messages are now placed on a\n  real, OS-thread-safe queue drained by a green worker on the hub, so\n  the callback is only ever invoked from the hub. No-op (direct call)\n  when eventlet is not patched.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a764b4f1fad2bd363e17297dc96f05ccbfd1e92d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a764b4f1fad2bd363e17297dc96f05ccbfd1e92d"}]},"branch":"refs/heads/master"},"75a89403c7637d9594df4d659fee16b251ea6bdc":{"kind":"REWORK","_number":8,"created":"2026-06-26 05:42:16.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/8"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-26 05:24:08.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched: the ASGI handler runs in a real OS thread, but\n  the RPC server\u0027s eventlet executor must only be poked from the hub.\n  Calling the server callback directly from the listener thread woke a\n  parked green worker via a cross-thread greenlet switch and raised\n  greenlet.error (observed as a cinder-scheduler ack timeout that\n  crash-looped cinder-volume). Incoming messages are now placed on a\n  real, OS-thread-safe queue drained by a green worker on the hub, so\n  the callback is only ever invoked from the hub. No-op (direct call)\n  when eventlet is not patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/75a89403c7637d9594df4d659fee16b251ea6bdc"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/75a89403c7637d9594df4d659fee16b251ea6bdc"}]},"branch":"refs/heads/master"},"dc822e49a32de4af7b015841704b5d00e8a1792a":{"kind":"REWORK","_number":9,"created":"2026-06-26 21:11:29.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/9","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/9","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/9"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-26 21:06:44.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched: the ASGI handler runs in a real OS thread, but\n  the RPC server\u0027s eventlet executor must only be poked from the hub.\n  Calling the server callback directly from the listener thread woke a\n  parked green worker via a cross-thread greenlet switch and raised\n  greenlet.error (observed as a cinder-scheduler ack timeout that\n  crash-looped cinder-volume). Incoming messages are now placed on a\n  real, OS-thread-safe queue drained by a green worker on the hub, so\n  the callback is only ever invoked from the hub. No-op (direct call)\n  when eventlet is not patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/dc822e49a32de4af7b015841704b5d00e8a1792a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/dc822e49a32de4af7b015841704b5d00e8a1792a"}]},"branch":"refs/heads/master"},"dc9d7432a38f88e42c22e5ba4d35a54b3b9b47ec":{"kind":"REWORK","_number":10,"created":"2026-06-26 23:22:34.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/10","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/10","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/10 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/10 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/10 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/10"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-26 22:02:55.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and Hypercorn ASGI server","message":"HTTP-RPC driver: httpx client and Hypercorn ASGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client and the Flask/cheroot WSGI server\nwith an httpx client and a bare ASGI application served by Hypercorn.\nThis enables HTTP/2, persistent connection pooling, and mutual TLS,\nand removes the Flask/Flask-RESTful/cheroot dependency stack.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nServer (Flask/cheroot -\u003e Hypercorn ASGI):\n- A bare ASGI application replaces the Flask Blueprint / Flask-RESTful\n  app (routes: /call, /cast, /cast_fanout, /healthcheck).\n- Hypercorn serves the app with native HTTP/2; when http2 is enabled,\n  HTTP/1.1 is disabled so httpcore always negotiates an HTTP/2\n  connection regardless of ALPN.\n- fd:// socket binding: the listener binds the socket itself and hands\n  the open fd to Hypercorn, so the port is never released between the\n  probe and the server start (no TOCTOU race). The bind address family\n  is resolved with getaddrinfo so IPv6 bind hosts work, and the socket\n  is put in the listening state before handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched: the ASGI handler runs in a real OS thread, but\n  the RPC server\u0027s eventlet executor must only be poked from the hub.\n  Calling the server callback directly from the listener thread woke a\n  parked green worker via a cross-thread greenlet switch and raised\n  greenlet.error (observed as a cinder-scheduler ack timeout that\n  crash-looped cinder-volume). Incoming messages are now placed on a\n  real, OS-thread-safe queue drained by a green worker on the hub, so\n  the callback is only ever invoked from the hub. No-op (direct call)\n  when eventlet is not patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  ASGI app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger.\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful, cheroot.\nDependencies added: hypercorn\u003e\u003d0.17.0.\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/dc9d7432a38f88e42c22e5ba4d35a54b3b9b47ec"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/dc9d7432a38f88e42c22e5ba4d35a54b3b9b47ec"}]},"branch":"refs/heads/master"},"e222cca796d10a57cdfe8016be9a7d4a71a1d44f":{"kind":"REWORK","_number":11,"created":"2026-06-30 06:48:10.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/11","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/11","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/11 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/11 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/11 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/11"}}},"commit":{"parents":[{"commit":"cd8fff240c5965bb8c1cb1ced5f44b4bea661845","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/cd8fff240c5965bb8c1cb1ced5f44b4bea661845"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-30 06:34:37.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e222cca796d10a57cdfe8016be9a7d4a71a1d44f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e222cca796d10a57cdfe8016be9a7d4a71a1d44f"}]},"branch":"refs/heads/master"},"04dfa52321c7e291f776a251649fcb27f7e99789":{"kind":"TRIVIAL_REBASE","_number":12,"created":"2026-07-01 07:00:07.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/12","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/12","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/12 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/12 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/12 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/12"}}},"commit":{"parents":[{"commit":"c1dea48e678e06c309903aaa8bf280e18a04cb3f","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c1dea48e678e06c309903aaa8bf280e18a04cb3f"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-01 06:57:54.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/04dfa52321c7e291f776a251649fcb27f7e99789"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/04dfa52321c7e291f776a251649fcb27f7e99789"}]},"branch":"refs/heads/master"},"649fbfa3b0a97bbbe0c9c31bf36f51aa73c9b415":{"kind":"REWORK","_number":13,"created":"2026-07-08 05:39:03.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/13","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/13","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/13 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/13 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/13 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/13"}}},"commit":{"parents":[{"commit":"a1a3631674fdef9de478fc6b42e08a5d5376150f","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a1a3631674fdef9de478fc6b42e08a5d5376150f"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-08 05:38:56.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/649fbfa3b0a97bbbe0c9c31bf36f51aa73c9b415"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/649fbfa3b0a97bbbe0c9c31bf36f51aa73c9b415"}]},"branch":"refs/heads/master"},"a729a5460e0f36b2bf7b20f9bb28935ff2d76db8":{"kind":"REWORK","_number":14,"created":"2026-07-08 06:19:36.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/14","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/14","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/14 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/14 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/14 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/14"}}},"commit":{"parents":[{"commit":"a1a3631674fdef9de478fc6b42e08a5d5376150f","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a1a3631674fdef9de478fc6b42e08a5d5376150f"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-08 06:19:05.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a729a5460e0f36b2bf7b20f9bb28935ff2d76db8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a729a5460e0f36b2bf7b20f9bb28935ff2d76db8"}]},"branch":"refs/heads/master"},"a9801a83e745389c061f825b03ca753d374b2965":{"kind":"REWORK","_number":15,"created":"2026-07-08 09:17:33.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/15","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/15","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/15 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/15 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/15 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/15"}}},"commit":{"parents":[{"commit":"9dd590684cffc1c6a6ec252b672f4d2ee29335fc","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/9dd590684cffc1c6a6ec252b672f4d2ee29335fc"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-08 09:07:23.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a9801a83e745389c061f825b03ca753d374b2965"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/a9801a83e745389c061f825b03ca753d374b2965"}]},"branch":"refs/heads/master"},"c8ef389a9a3f620a243486d8b40c06fdf4b034d1":{"kind":"REWORK","_number":16,"created":"2026-07-08 18:29:40.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/16","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/16","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/16 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/16 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/16 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/16"}}},"commit":{"parents":[{"commit":"03578246912d2752bb7c35f9074a29fd078349ee","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/03578246912d2752bb7c35f9074a29fd078349ee"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-08 18:29:19.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c8ef389a9a3f620a243486d8b40c06fdf4b034d1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c8ef389a9a3f620a243486d8b40c06fdf4b034d1"}]},"branch":"refs/heads/master"},"28fe3ec83a069e27b03718af3f0905ba1081c7a0":{"kind":"TRIVIAL_REBASE","_number":17,"created":"2026-07-08 18:52:48.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/17","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/17","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/17 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/17 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/17 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/17"}}},"commit":{"parents":[{"commit":"7b7e96a3e8f0c93bc04ee947863adaa05f107393","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/7b7e96a3e8f0c93bc04ee947863adaa05f107393"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-08 18:47:39.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/28fe3ec83a069e27b03718af3f0905ba1081c7a0"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/28fe3ec83a069e27b03718af3f0905ba1081c7a0"}]},"branch":"refs/heads/master"},"818a7e91304e09b265370316247727bcebd0c68d":{"kind":"REWORK","_number":18,"created":"2026-07-09 05:22:05.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/18","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/18","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/18 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/18 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/18 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/18"}}},"commit":{"parents":[{"commit":"2d79a928986a8505810321d525c332b69aad1094","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2d79a928986a8505810321d525c332b69aad1094"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-09 05:01:19.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/818a7e91304e09b265370316247727bcebd0c68d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/818a7e91304e09b265370316247727bcebd0c68d"}]},"branch":"refs/heads/master"},"e0b6cb1454c9b2152773e1cc03797c8c7c9186ae":{"kind":"REWORK","_number":19,"created":"2026-07-09 10:18:00.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/19","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/19","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/19 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/19 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/19 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/19"}}},"commit":{"parents":[{"commit":"02aafab04cc26ab40dcc00f6dda37b73051f79d3","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/02aafab04cc26ab40dcc00f6dda37b73051f79d3"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-09 10:15:29.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e0b6cb1454c9b2152773e1cc03797c8c7c9186ae"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e0b6cb1454c9b2152773e1cc03797c8c7c9186ae"}]},"branch":"refs/heads/master"},"6656af3b81ac40cd7ee8eac3e203a57d7459e7b6":{"kind":"REWORK","_number":20,"created":"2026-07-09 10:47:31.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/20","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/20","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/20 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/20 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/20 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/20"}}},"commit":{"parents":[{"commit":"af08014e5d34149f650d479b8904956a3795e7ca","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/af08014e5d34149f650d479b8904956a3795e7ca"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-09 10:46:48.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/6656af3b81ac40cd7ee8eac3e203a57d7459e7b6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/6656af3b81ac40cd7ee8eac3e203a57d7459e7b6"}]},"branch":"refs/heads/master"},"2240c6cb371c3bffc6dceeee4eb16099d4af5490":{"kind":"REWORK","_number":21,"created":"2026-07-09 23:23:38.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/21","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/21","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/21 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/21 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/21 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/21"}}},"commit":{"parents":[{"commit":"dd45bf492c249a36cb20bb3585b7ec8990ab64ab","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/dd45bf492c249a36cb20bb3585b7ec8990ab64ab"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-09 23:21:10.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2240c6cb371c3bffc6dceeee4eb16099d4af5490"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2240c6cb371c3bffc6dceeee4eb16099d4af5490"}]},"branch":"refs/heads/master"},"f803eb63f6bbc5e69cae0385cfc6c80d8cc4494c":{"kind":"REWORK","_number":22,"created":"2026-07-10 02:30:33.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/22","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/22","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/22 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/22 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/22 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/22"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 02:26:53.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/f803eb63f6bbc5e69cae0385cfc6c80d8cc4494c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/f803eb63f6bbc5e69cae0385cfc6c80d8cc4494c"}]},"branch":"refs/heads/master"},"5940a6c9d36709445a90ff2c6efbf15f600beb09":{"kind":"REWORK","_number":23,"created":"2026-07-10 04:51:06.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/23","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/23","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/23 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/23 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/23 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/23"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 04:48:57.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/5940a6c9d36709445a90ff2c6efbf15f600beb09"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/5940a6c9d36709445a90ff2c6efbf15f600beb09"}]},"branch":"refs/heads/master"},"71f19144fab782717e8d067c5520a87b3ae3c901":{"kind":"REWORK","_number":24,"created":"2026-07-10 17:57:46.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/24","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/24","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/24 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/24 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/24 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/24"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 17:56:48.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/71f19144fab782717e8d067c5520a87b3ae3c901"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/71f19144fab782717e8d067c5520a87b3ae3c901"}]},"branch":"refs/heads/master"},"4895cc7d413abae91774c1d6c136861c07eb9296":{"kind":"REWORK","_number":25,"created":"2026-07-10 18:13:55.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/25","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/25","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/25 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/25 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/25 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/25"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 18:13:27.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4895cc7d413abae91774c1d6c136861c07eb9296"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4895cc7d413abae91774c1d6c136861c07eb9296"}]},"branch":"refs/heads/master"},"e295c5334e1a0cd8ee1cc10034c2c44a93885dab":{"kind":"REWORK","_number":26,"created":"2026-07-10 20:28:49.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/26","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/26","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/26 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/26 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/26 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/26"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 20:28:43.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e295c5334e1a0cd8ee1cc10034c2c44a93885dab"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e295c5334e1a0cd8ee1cc10034c2c44a93885dab"}]},"branch":"refs/heads/master"},"c84214064e1b9754fa8a65f3d62a05c9bc9bcee8":{"kind":"REWORK","_number":27,"created":"2026-07-10 23:06:58.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/27","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/27","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/27 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/27 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/27 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/27"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-10 23:06:49.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c84214064e1b9754fa8a65f3d62a05c9bc9bcee8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c84214064e1b9754fa8a65f3d62a05c9bc9bcee8"}]},"branch":"refs/heads/master"},"870bdb5a8db888d068a03c431fe2fafb7ecc43c5":{"kind":"REWORK","_number":28,"created":"2026-07-11 01:26:56.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/28","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/28","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/28 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/28 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/28 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/28"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 01:26:46.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/870bdb5a8db888d068a03c431fe2fafb7ecc43c5"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/870bdb5a8db888d068a03c431fe2fafb7ecc43c5"}]},"branch":"refs/heads/master"},"73928880f69faf140ea5f9a275cbda13d9461137":{"kind":"REWORK","_number":29,"created":"2026-07-11 02:35:08.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/29","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/29","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/29 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/29 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/29 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/29"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 02:34:59.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/73928880f69faf140ea5f9a275cbda13d9461137"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/73928880f69faf140ea5f9a275cbda13d9461137"}]},"branch":"refs/heads/master"},"c5c9775c9287936990a0d3a32bc4eff4bf80fa5e":{"kind":"REWORK","_number":30,"created":"2026-07-11 08:11:42.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/30","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/30","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/30 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/30 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/30 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/30"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 08:10:51.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c5c9775c9287936990a0d3a32bc4eff4bf80fa5e"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/c5c9775c9287936990a0d3a32bc4eff4bf80fa5e"}]},"branch":"refs/heads/master"},"50dd95cd796de7a839afc716b1ccac2524392fa6":{"kind":"REWORK","_number":31,"created":"2026-07-11 20:02:12.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/31","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/31","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/31 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/31 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/31 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/31"}}},"commit":{"parents":[{"commit":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/036ffd0bcf9d79420f2b545cbc8d644840c35ec6"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 20:01:44.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/50dd95cd796de7a839afc716b1ccac2524392fa6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/50dd95cd796de7a839afc716b1ccac2524392fa6"}]},"branch":"refs/heads/master"},"274ea8b9d8f0099a1b342eeb9866747095a1f4f4":{"kind":"TRIVIAL_REBASE","_number":32,"created":"2026-07-11 21:36:46.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/32","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/32","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/32 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/32 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/32 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/32"}}},"commit":{"parents":[{"commit":"4fadf355c46d2568aa169b6bb259b6947551d0f8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4fadf355c46d2568aa169b6bb259b6947551d0f8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 21:36:46.000000000","tz":0},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/274ea8b9d8f0099a1b342eeb9866747095a1f4f4"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/274ea8b9d8f0099a1b342eeb9866747095a1f4f4"}]},"branch":"refs/heads/master","description":"Rebase","conflicts":{"base":"036ffd0bcf9d79420f2b545cbc8d644840c35ec6","ours":"50dd95cd796de7a839afc716b1ccac2524392fa6","theirs":"4fadf355c46d2568aa169b6bb259b6947551d0f8","merge_strategy":"recursive","contains_conflicts":false}},"99c5a6e35cda96d59716eb573a018ed59cd158f6":{"kind":"REWORK","_number":33,"created":"2026-07-11 23:31:49.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/33","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/33","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/33 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/33 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/33 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/33"}}},"commit":{"parents":[{"commit":"4fadf355c46d2568aa169b6bb259b6947551d0f8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4fadf355c46d2568aa169b6bb259b6947551d0f8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-11 23:31:07.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/99c5a6e35cda96d59716eb573a018ed59cd158f6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/99c5a6e35cda96d59716eb573a018ed59cd158f6"}]},"branch":"refs/heads/master"},"3aecc6dcf572220596c5207fd327ede328d1c619":{"kind":"REWORK","_number":34,"created":"2026-07-12 03:18:51.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/34","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/34","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/34 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/34 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/34 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/34"}}},"commit":{"parents":[{"commit":"4fadf355c46d2568aa169b6bb259b6947551d0f8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4fadf355c46d2568aa169b6bb259b6947551d0f8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-12 03:17:12.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/3aecc6dcf572220596c5207fd327ede328d1c619"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/3aecc6dcf572220596c5207fd327ede328d1c619"}]},"branch":"refs/heads/master"},"d3f170d58226be961c2ae1b10cf0363249541809":{"kind":"REWORK","_number":35,"created":"2026-07-12 05:48:53.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/35","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/35","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/35 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/35 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/35 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/35"}}},"commit":{"parents":[{"commit":"4fadf355c46d2568aa169b6bb259b6947551d0f8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4fadf355c46d2568aa169b6bb259b6947551d0f8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-12 05:48:43.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d3f170d58226be961c2ae1b10cf0363249541809"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d3f170d58226be961c2ae1b10cf0363249541809"}]},"branch":"refs/heads/master"},"1a2593bf23e8e580c5c9a5434b12eb20cca7de5c":{"kind":"REWORK","_number":36,"created":"2026-07-12 09:23:40.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/36","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/36","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/36 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/36 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/36 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/36"}}},"commit":{"parents":[{"commit":"4fadf355c46d2568aa169b6bb259b6947551d0f8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4fadf355c46d2568aa169b6bb259b6947551d0f8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-12 09:23:29.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/1a2593bf23e8e580c5c9a5434b12eb20cca7de5c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/1a2593bf23e8e580c5c9a5434b12eb20cca7de5c"}]},"branch":"refs/heads/master"},"4d8b9b5aa14cc027788efd5ab614358a6f95b559":{"kind":"REWORK","_number":37,"created":"2026-07-17 05:41:12.000000000","uploader":{"_account_id":10273,"name":"Adam Harwell","email":"flux.adam@gmail.com","username":"rm_you"},"ref":"refs/changes/16/993316/37","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/16/993316/37","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/37 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/37 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/37 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/16/993316/37"}}},"commit":{"parents":[{"commit":"3f05700d7b9f2c9fb19ee5469521d8046f67cab8","subject":"HTTP-RPC driver: various fixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/3f05700d7b9f2c9fb19ee5469521d8046f67cab8"}]}],"author":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-06-22 22:45:56.000000000","tz":-240},"committer":{"name":"Adam Harwell","email":"flux.adam@gmail.com","date":"2026-07-17 05:39:49.000000000","tz":-240},"subject":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server","message":"HTTP-RPC driver: httpx client and pluggable ASGI/WSGI server\n\nModernize the HTTP transport for the oslo.messaging HTTP-RPC driver,\nreplacing the requests-based client with httpx and replacing the\nFlask application with a small framework-free app that can be served\nby either of two interchangeable, driver-launched in-process servers:\nHypercorn (ASGI, the default, adds HTTP/2) or cheroot (WSGI, HTTP/1.1).\nFlask and Flask-RESTful are dropped; cheroot is retained so the\nexisting WSGI deployment model keeps working. This enables HTTP/2,\npersistent connection pooling, and mutual TLS without removing the\nWSGI option operators already rely on.\n\nClient (requests -\u003e httpx):\n- Persistent connection pooling and HTTP/2 support (http2 option).\n- SNITransport sets the TLS server name when connecting by IP address\n  so certificate verification works against pod IPs.\n- Mutual TLS: ssl_ca_file (server verification / custom CA),\n  ssl_client_certfile and ssl_client_keyfile (client auth).\n- Fork-safe client: lazily created and keyed by PID, so connection\n  pools are never shared across a fork (e.g. gunicorn pre-fork\n  workers). The PID check runs before client selection so the per-SNI\n  clients are recreated on fork too, not just the default client.\n- Fanout retry with exponential backoff for transient connection\n  errors (SSL EOF, connection reset).\n\nApplication (Flask -\u003e framework-free):\n- A bare app (no Flask) handles the routes /call, /cast, /cast_fanout\n  and /healthcheck. The framework-free request handling, auth\n  (constant-time token compare, fail-closed), message construction,\n  reply decoding and the max_concurrent_requests in-flight gate live\n  in a shared app_common module used by both the ASGI and WSGI apps,\n  so the two servers expose identical behaviour.\n\nServer back ends (Flask/cheroot -\u003e pluggable Hypercorn/cheroot):\n- The server option selects the in-process server: \u0027hypercorn\u0027 (the\n  default) serves the ASGI app; \u0027cheroot\u0027 serves the WSGI app. Both\n  are launched by the driver itself (each binds its own socket and\n  registers its host/port with the service broker); neither runs\n  behind an external gunicorn/uWSGI. The broadcaster honours the same\n  option. The cheroot path is the pre-existing WSGI deployment model,\n  preserved for backward compatibility -- an operator upgrading from\n  the original Flask/cheroot driver can set server\u003dcheroot and keep\n  HTTP/1.1 WSGI; everyone else gets Hypercorn transparently.\n- The WSGI app needs none of the eventlet/asyncio machinery below: its\n  handler runs on the cheroot worker thread and blocks directly on the\n  message\u0027s threading.Event (get_result / wait_for_acknowledge), which\n  is safe to set from the hub and wait on from a foreign OS thread.\n- Hypercorn serves the ASGI app with native HTTP/2; when http2 is\n  enabled (now the default), HTTP/1.1 is disabled so httpcore always\n  negotiates an HTTP/2 connection regardless of ALPN. http2 only takes\n  effect with server\u003dhypercorn; with server\u003dcheroot the driver logs a\n  warning at listener start and serves HTTP/1.1 (clients fall back via\n  ALPN automatically).\n- fd:// socket binding (Hypercorn path): the listener binds the socket\n  itself and hands the open fd to Hypercorn, so the port is never\n  released between the probe and the server start (no TOCTOU race).\n  The bind address family is resolved with getaddrinfo so IPv6 bind\n  hosts work, and the socket is put in the listening state before\n  handoff.\n- Blocking RPC dispatch runs in a worker thread; the reply/ack events\n  use the real (non-eventlet) threading primitives, and the Hypercorn\n  loop runs in a real OS thread for eventlet compatibility. (Under\n  eventlet the reply wakeup is delivered on the hub instead -- see\n  below.)\n- The listener\u0027s asyncio loop is given a selector built from\n  eventlet\u0027s original (non-green) select when the process is\n  monkey-patched, so a second listener started in a real OS thread\n  cannot drive eventlet\u0027s hub and raise greenlet.error. No-op when\n  eventlet is not patched.\n- Inbound message hand-off is marshalled onto the eventlet hub thread\n  when monkey-patched. Both server back ends run their request handler\n  off the hub (Hypercorn on its asyncio-loop thread, cheroot on a\n  worker-pool thread), but the RPC server\u0027s eventlet executor must\n  only be poked from the hub. Calling the server callback directly\n  from the off-hub thread woke a parked green worker via a cross-thread\n  greenlet switch and raised greenlet.error (observed as a\n  cinder-scheduler ack timeout that crash-looped cinder-volume).\n  Incoming messages are now placed on a real, OS-thread-safe queue\n  drained by a green worker on the hub, so the callback is only ever\n  invoked from the hub. No-op (direct call) when eventlet is not\n  patched.\n- Reply/ack delivery is also marshalled onto the eventlet hub when\n  monkey-patched, completing the inbound fix above. Waiting for the\n  dispatch result via asyncio.to_thread parked a waiter on the green\n  executor\u0027s internal semaphore from a foreign OS thread; when that\n  semaphore later woke the waiter from a hub timer it performed a\n  cross-thread greenlet switch and raised greenlet.error, which\n  permanently corrupted the executor\u0027s concurrency semaphore and\n  wedged the service (observed as cinder-volume processing zero RPC\n  work under sustained attachment_update load). The ASGI handler now\n  arms an asyncio.Future on its own listener loop before dispatch and\n  awaits it; reply()/acknowledge() wake that Future via\n  loop.call_soon_threadsafe, so no foreign-thread waiter is ever\n  parked on the executor semaphore. The real threading.Event still\n  carries the reply data; the Future only carries the wakeup. The\n  eventlet-off path keeps using asyncio.to_thread unchanged.\n- The outbound httpx client is confined to one connection pool per\n  OS-thread hub when monkey-patched. A single shared client is touched\n  both from the hub MainThread (green dispatch workers) and from each\n  Hypercorn listener\u0027s own real OS thread; httpcore guards its pool\n  with green Lock/Event primitives, so a waiter parked on one hub and\n  woken from another hub\u0027s timer performed a cross-thread greenlet\n  switch and raised greenlet.error, corrupting the hub timer wheel\n  (reliably reachable once mTLS made the TLS handshake hold the pool\n  lock for the duration of the connect). The client cache is now keyed\n  by (target, thread-ident) under eventlet so each hub gets its own\n  pool and the green pool primitives never span hubs. The eventlet-off\n  path uses the bare key and is unchanged.\n- max_concurrent_requests is enforced by an in-flight counter in the\n  app (HTTP 503 when full; 0 \u003d unlimited).\n- Hypercorn\u0027s TCPServer._close is patched to swallow the SSL-shutdown\n  TimeoutError that kubelet HTTPS probes trigger (Hypercorn path only).\n\nService discovery:\n- Add a _create_service_broker() factory selected by the\n  service_broker_backend option (currently \u0027consul\u0027); ConsulOperator\n  remains the broker. advertise_host lets a service register an\n  address that differs from its bind host (NAT, reverse proxy, or\n  container deployments).\n\nDependencies removed: Flask, Flask-RESTful.\nDependencies added: hypercorn\u003e\u003d0.17.0 (cheroot is retained).\nDependencies changed: httpx -\u003e httpx[http2].\n\nGenerated-By: claude-opus-4-6 (OpenCode)\nChange-Id: I8b0d949c148e01366cf38789a009c4aef7b14843\nSigned-off-by: flux.adam@gmail.com\nSigned-off-by: Adam Harwell \u003cflux.adam@gmail.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4d8b9b5aa14cc027788efd5ab614358a6f95b559"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/4d8b9b5aa14cc027788efd5ab614358a6f95b559"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"},{"label":"Backport-Candidate","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Backport-Candidate","description":"Backport candidate status","status":"NOT_APPLICABLE","is_legacy":false,"applicability_expression_result":{"fulfilled":false,"status":"FAIL"},"submittability_expression_result":{"expression":"is:true","fulfilled":true,"status":"NOT_EVALUATED","passing_atoms":[],"failing_atoms":[],"atom_explanations":{}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}