)]}'
{"id":"openstack%2Fopenstack-helm~1010873","triplet_id":"openstack%2Fopenstack-helm~master~I69e20e71fb1e8c0bbc585368f14e804ab8c81689","project":"openstack/openstack-helm","branch":"master","attention_set":{},"removed_from_attention_set":{"7102":{"account":{"_account_id":7102,"name":"Thomas Bechtold","email":"thomas.bechtold@chainguard.dev","username":"toabctl"},"last_update":"2026-10-16 17:27:26.000000000","reason":"Change was submitted"}},"hashtags":[],"change_id":"I69e20e71fb1e8c0bbc585368f14e804ab8c81689","subject":"Keep the compute-kit RabbitMQ off Ceph","status":"MERGED","created":"2026-10-16 12:43:19.000000000","updated":"2026-10-16 17:29:17.000000000","submitted":"2026-10-16 17:27:26.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"1010873","meta_rev_id":"521b12c7f4a5e4276810214222507710be51df5b","_number":1010873,"virtual_id_number":1010873,"owner":{"_account_id":7102,"name":"Thomas Bechtold","email":"thomas.bechtold@chainguard.dev","username":"toabctl"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},{"value":0,"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2026-10-16 17:27:26.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},"all":[{"value":2,"date":"2026-10-16 15:34:00.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},{"value":2,"date":"2026-10-16 15:26:12.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},"all":[{"value":1,"date":"2026-10-16 15:34:00.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},{"value":0,"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},{"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2026-10-16 14:40:45.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2026-10-16 15:26:12.000000000","updated_by":{"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},"reviewer":{"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},"state":"REVIEWER"},{"updated":"2026-10-16 15:34:00.000000000","updated_by":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},"reviewer":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},"state":"REVIEWER"}],"messages":[{"id":"cc3ff5b7edbfd675c47a7cbeda4db538eecfc76e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7102,"name":"Thomas Bechtold","email":"thomas.bechtold@chainguard.dev","username":"toabctl"},"date":"2026-10-16 12:43:19.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"f84b0b1164e106d08b219af0eb3289a06d16c248","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-10-16 14:40:45.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/dc9dc05c143a4a2b98b40b16a41d4f9c\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/a6599379735842c2aae11ee54a3b6270 : SUCCESS in 3m 43s\n- openstack-helm-linter https://zuul.opendev.org/t/openstack/build/b85938bc3da34fb3b1ced930d8593134 : SUCCESS in 3m 12s\n- openstack-helm-pre-commit https://zuul.opendev.org/t/openstack/build/b96a326b2c1c406da833cbd8eada62a3 : SUCCESS in 2m 41s\n- openstack-helm-release-notes https://zuul.opendev.org/t/openstack/build/ce4992f9a03d4c4c90622b85608cfbbe : SUCCESS in 1m 49s\n- openstack-helm-build-charts https://zuul.opendev.org/t/openstack/build/bda8c452ab174ed1802f9ba03ab8225e : SUCCESS in 2m 53s\n- openstack-helm-compute-kit-2025-2-ubuntu_noble https://zuul.opendev.org/t/openstack/build/c44d1d51545140e2b64fd3a00fe3a049 : SUCCESS in 1h 42m 24s\n- openstack-helm-compute-kit-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/da403c61e1064b46a80e694ac1cb7bbe : SUCCESS in 1h 28m 26s\n- openstack-helm-tls-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/2ae622bd17db406b948ee74b535750df : SUCCESS in 1h 02m 11s\n- openstack-helm-ovn-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/123690dac7de40c7851f72749fffb72b : SUCCESS in 45m 56s\n- openstack-helm-dpdk-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/a636eb07e2a5464085d974219cabb2c3 : SUCCESS in 56m 54s\n- openstack-helm-crd-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/f3c34f96859b4a0fae1cc1ff79c05a9f : SUCCESS in 1h 34m 14s\n- openstack-helm-operators-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/e86bcb6f6dd146a9aa347f89d786c6d8 : SUCCESS in 1h 38m 00s\n- openstack-helm-compute-kit-2026-2-ubuntu_resolute https://zuul.opendev.org/t/openstack/build/2e82331a83ce48e799f14ce8e4f7e1ef : SUCCESS in 1h 15m 02s","accounts_in_message":[],"_revision_number":1},{"id":"1645b00f3709b23c2078934f70cb790391841407","author":{"_account_id":7156,"name":"Mathieu Gagné","email":"mgagne@calavera.ca","username":"mgagne"},"date":"2026-10-16 15:26:12.000000000","message":"Patch Set 1: Code-Review+2","accounts_in_message":[],"_revision_number":1},{"id":"a048b7c17dd1ecfb552693a31f4279d067714d73","author":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"},"date":"2026-10-16 15:34:00.000000000","message":"Patch Set 1: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":1},{"id":"c2ba9f7c887cfd100e6c18da406daed44fbc6657","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-10-16 15:34:15.000000000","message":"Patch Set 1: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":1},{"id":"054129ed2bbed04b470dda0996128b51330e61ec","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-10-16 17:27:26.000000000","message":"Patch Set 1: Verified+2\n\nBuild succeeded (gate pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/64839f4fac1c45ef9c9edfd6c67343a1\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/47bc03cb57004a7687317c344c144f65 : SUCCESS in 2m 45s\n- openstack-helm-linter https://zuul.opendev.org/t/openstack/build/d85155cc67cb4b339f77c2cc6134c827 : SUCCESS in 3m 09s\n- openstack-helm-release-notes https://zuul.opendev.org/t/openstack/build/000ab6ce440d4b73887ab0c426cb3864 : SUCCESS in 1m 56s\n- openstack-helm-compute-kit-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/1efc78c2b11f41de8fc1815f1153c0ec : SUCCESS in 1h 09m 37s\n- openstack-helm-logging https://zuul.opendev.org/t/openstack/build/6053462df696471782ffd2cfae32a4bd : SUCCESS in 23m 39s\n- openstack-helm-monitoring https://zuul.opendev.org/t/openstack/build/7fcd8266420943bcb9c4e716c25d5c25 : SUCCESS in 33m 31s","accounts_in_message":[],"_revision_number":1},{"id":"da6608d8a8f6c2a3025f37d0249222a5ceaf51dd","tag":"autogenerated:gerrit:merged","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-10-16 17:27:26.000000000","message":"Change has been successfully merged","accounts_in_message":[],"_revision_number":1},{"id":"521b12c7f4a5e4276810214222507710be51df5b","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-10-16 17:29:17.000000000","message":"Patch Set 1:\n\nBuild succeeded (promote pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/2895bbe3d64d41c2a3671d53b0a76a0c\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/037d2c0472914d8392585790e89da9da : SUCCESS in 45s","accounts_in_message":[],"_revision_number":1}],"current_revision_number":1,"current_revision":"0b6213c1dc5bb5073fd25f180b41a8d5e188c7ac","revisions":{"0b6213c1dc5bb5073fd25f180b41a8d5e188c7ac":{"kind":"REWORK","_number":1,"created":"2026-10-16 12:43:19.000000000","uploader":{"_account_id":7102,"name":"Thomas Bechtold","email":"thomas.bechtold@chainguard.dev","username":"toabctl"},"ref":"refs/changes/73/1010873/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/73/1010873/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/73/1010873/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/73/1010873/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/73/1010873/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/73/1010873/1"}}},"commit":{"parents":[{"commit":"ec221d6a59e4b0b361fc6d779626ac59c0b6f1c4","subject":"Merge \"masakari: render the db-sync job with helm-toolkit\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/ec221d6a59e4b0b361fc6d779626ac59c0b6f1c4"}]}],"author":{"name":"Thomas Bechtold","email":"thomasbechtold@jpberlin.de","date":"2026-10-16 12:43:13.000000000","tz":120},"committer":{"name":"Thomas Bechtold","email":"thomasbechtold@jpberlin.de","date":"2026-10-16 12:43:13.000000000","tz":120},"subject":"Keep the compute-kit RabbitMQ off Ceph","message":"Keep the compute-kit RabbitMQ off Ceph\n\ncompute-kit jobs on OVH nodes fail because RabbitMQ stops serving a\nvhost or a set of quorum queues partway through the deployment, and it\ndoes not recover:\n\n  access to vhost \u0027cinder\u0027 refused for user \u0027cinder\u0027: vhost \u0027cinder\u0027 is down\n  failed consuming from quorum queue \u0027reply_nova-scheduler-0:health-probe-readiness:1\u0027 in vhost \u0027nova\u0027: noproc\n\nBoth happened on the same patch set of 1010828, which only touches a\nglance value. The first left cinder-volume down, the second kept\nnova-scheduler from becoming ready, and neutron\u0027s queues failed the\nsame way. Of the last 40 or so runs of each of the 2025.2 and 2026.1\ncompute-kit jobs, all five failures were on OVH nodes.\n\nSince 888aa486f the broker keeps its mnesia directory on an RBD volume\nwhen Ceph is deployed. The OSDs sit on loop devices, and in both failed\njobs their BlueStore commits stalled for 15 to 40 seconds, with slow\nops and monitors dropping out of quorum at the same time:\n\n  bluestore(/var/lib/ceph/osd/ceph-4) log_latency slow operation observed for kv_commit, latency \u003d 39.351364136s\n\nThe cinder vhost was added just as one of these stalls began and never\nstarted its message store. The nova and neutron queues failed during\nanother one. The earlier patch set of 1010828 passed on raxflex with no\ncommit above 5s.\n\nThe node disks themselves look fine. etcd on the primary node writes\nto the same disk as that node\u0027s OSD and logged no fdatasync slower\nthan a second in either job, while the OSD stalled for 20 and 40\nseconds.\n\nAdd osh_rabbitmq_storage, \"pvc\" or \"local\" like osh_glance_backend,\nfollowing osh_ceph_storage unless set, and pin it to \"local\" in\ndeploy-compute-kit.yaml. That is the host path the broker used before\n888aa486f. mariadb and glance stay on Ceph, so the CSI path is still\nexercised.\n\nThis should help, but it is not proven to be enough. There are only two\nfailed jobs to go on, the disk latency on the nodes running RabbitMQ\nwas not measured directly, and it is not clear why RabbitMQ never\nbrings the vhost or the queues back after a stall. The OVH failure rate\nafter this merges will tell.\n\nChecked by rendering the rabbitmq values with Ceph, without it, and\nwith osh_rabbitmq_storage set either way.\n\nAssisted-by: Claude Opus 5.5\nChange-Id: I69e20e71fb1e8c0bbc585368f14e804ab8c81689\nSigned-off-by: Thomas Bechtold \u003cthomasbechtold@jpberlin.de\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/0b6213c1dc5bb5073fd25f180b41a8d5e188c7ac"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/0b6213c1dc5bb5073fd25f180b41a8d5e188c7ac"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"CLOSED","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY","applied_by":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"}},{"label":"Workflow","status":"MAY","applied_by":{"_account_id":5890,"name":"Doug Goldstein","email":"cardoe@cardoe.com","username":"cardoe"}}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Verified\u003dMAX"],"failing_atoms":["label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Code-Review\u003dMAX"],"failing_atoms":["label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Workflow\u003dMAX"],"failing_atoms":["label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}