)]}'
{"id":"openstack%2Fcinder~989836","triplet_id":"openstack%2Fcinder~master~Icdd28affc73fd34491b656a68410dce8e46264d4","project":"openstack/cinder","branch":"master","attention_set":{"5997":{"account":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"last_update":"2026-05-28 05:09:50.000000000","reason":"Someone else replied on the change"}},"removed_from_attention_set":{"38702":{"account":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"last_update":"2026-06-19 20:01:44.000000000","reason":"removed on reply"}},"hashtags":[],"change_id":"Icdd28affc73fd34491b656a68410dce8e46264d4","subject":"Implement graceful shutdown for Cinder services","status":"NEW","created":"2026-05-22 21:29:51.000000000","updated":"2026-06-19 20:06:07.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"7505e3fdd3db75873508fe5d1b6bc19208675325","_number":989836,"virtual_id_number":989836,"owner":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":-1,"date":"2026-05-27 23:59:31.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Review-Priority":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Branch Freeze"," 0":"No Priority","+1":"Important Change","+2":"Gate Blocker Fix / Urgent Change"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"CC":[{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},{"_account_id":38252,"name":"HPE AlletraMP NVMeTCP CI","display_name":"HPE AlletraMP NVMeTCP CI","email":"yogeshwari.gitte-ext@hpe.com","username":"yogeshwarigitte"},{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2026-05-22 22:06:59.000000000","updated_by":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2026-05-22 22:28:55.000000000","updated_by":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"reviewer":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2026-05-22 22:34:42.000000000","updated_by":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"reviewer":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"state":"CC"},{"updated":"2026-05-22 23:02:03.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2026-05-22 23:10:33.000000000","updated_by":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"reviewer":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"state":"CC"},{"updated":"2026-05-22 23:18:29.000000000","updated_by":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"reviewer":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"state":"CC"},{"updated":"2026-05-22 23:32:51.000000000","updated_by":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"reviewer":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2026-05-23 00:55:24.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2026-05-27 21:45:50.000000000","updated_by":{"_account_id":38252,"name":"HPE AlletraMP NVMeTCP CI","display_name":"HPE AlletraMP NVMeTCP CI","email":"yogeshwari.gitte-ext@hpe.com","username":"yogeshwarigitte"},"reviewer":{"_account_id":38252,"name":"HPE AlletraMP NVMeTCP CI","display_name":"HPE AlletraMP NVMeTCP CI","email":"yogeshwari.gitte-ext@hpe.com","username":"yogeshwarigitte"},"state":"CC"},{"updated":"2026-06-19 19:51:51.000000000","updated_by":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"reviewer":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"state":"CC"}],"messages":[{"id":"7c02b1d3f7d8da188af4331fca1f19978251f219","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-22 21:29:51.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"813faf999aa1955b931e4fd0ef6547bdfabbbcfd","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-22 21:32:03.000000000","message":"Uploaded patch set 2: Commit message was updated.","accounts_in_message":[],"_revision_number":2},{"id":"2b1b47b8e6437a4c29c1f409a855ee4d54f5750c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-22 21:36:57.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"cfb5cffce9c42da5231e5d5d23331cac868ed062","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-22 22:06:59.000000000","message":"Patch Set 3:\n\nBuild failed.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/48a08e47004c4dcbb35100d79962876b : FAILURE in 29m 03s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":3},{"id":"57fe7f25b7e0c33d9372233371c4b76a4525d54c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-22 22:16:22.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"025da39eae41c3d31ca952eb96261726b3015a73","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-22 22:28:11.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"4c4db68224318e1a33aab61a812c39557488ff6a","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-22 22:28:55.000000000","message":"Patch Set 1:\n\nBuild Successful \n\n* quobyteci:  cinder_quobyteci : SUCCESS, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":1},{"id":"a257248fada6b9b4507289d0b09fa99dec12b43b","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-22 22:30:52.000000000","message":"Patch Set 2:\n\nBuild Successful \n\n* quobyteci:  cinder_quobyteci : SUCCESS, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":2},{"id":"5dd78b29ab42a7d191ef1d30ee2e1442650a6782","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-22 22:34:42.000000000","message":"Patch Set 1:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4226/.","accounts_in_message":[],"_revision_number":1},{"id":"debeaaa3b27051f231a07483128651705e18db97","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-22 22:37:45.000000000","message":"Patch Set 3:\n\nBuild Successful \n\n* quobyteci:  cinder_quobyteci : SUCCESS, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":3},{"id":"0120f9678c7f5846f8188875c7286a326a0b60ad","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-22 22:38:01.000000000","message":"Patch Set 1:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4050/.","accounts_in_message":[],"_revision_number":1},{"id":"d288d8229b8557e093ec8fcc9565b88c0b7b1a9f","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-22 23:02:03.000000000","message":"Patch Set 5:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/6b590fda5dca42558d78ab5edfea6ae4\n\n- openstack-tox-py311-arm64 https://zuul.opendev.org/t/openstack/build/da11c23e04874cf797a65a09699f5ddb : FAILURE in 25m 16s (non-voting)\n- openstack-tox-py313-arm64 https://zuul.opendev.org/t/openstack/build/83c7cf2ea52d440ebe17504bf1a44c4c : FAILURE in 22m 32s (non-voting)\n- openstack-tox-py314-arm64 https://zuul.opendev.org/t/openstack/build/89da1536e6624680a0a0331893dead93 : FAILURE in 31m 26s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"7a822779dd51c0d5162c17133b59f664516a5bd6","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-22 23:02:21.000000000","message":"Patch Set 5:\n\nBuild failed.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/4354d4c6782143819402d99e13bf9683 : FAILURE in 28m 42s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":5},{"id":"4be3902bd437398bd3cfade21dbfa4357915f836","tag":"autogenerated:zuul:check","author":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"date":"2026-05-22 23:10:33.000000000","message":"Patch Set 5:\n\nBuild failed.\n\n- hpe-fc-driver build/c860c0c0722a4f02a57b7943524e632b : FAILURE in 40m 46s\n\nWarning:\n  FC CI log: https://github.com/hpe-storage/hpe-ci-log/tree/master/logs/989836","accounts_in_message":[],"_revision_number":5},{"id":"acb38346ff6522e13854969dee4e545ca6e95d3a","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-22 23:12:13.000000000","message":"Patch Set 4:\n\nBuild Failed \n\n* quobyteci:  cinder_quobyteci : FAILURE, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":4},{"id":"dab8d136cf06767846df275738f0c5bf60e6bcdb","author":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"date":"2026-05-22 23:18:29.000000000","message":"Patch Set 5:\n\nBuild Succeeded: 3par-fc-driver: Logs: https://github.com/hpe-storage/hpe-3parfc-ci/tree/main/new_logs/989836 : SUCCESS","accounts_in_message":[],"_revision_number":5},{"id":"c09b4452e2c5e08c69c597efd92390d5b2c32919","author":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"date":"2026-05-22 23:32:51.000000000","message":"Patch Set 5:\n\nBuild failed.\n\n- cinder-storpool-tempest build/cf746db94078443ea0c46932ecc3b64d : FAILURE in 51m 07s\n- cinder-storpool-tempest-iscsi-multipath build/26b91e3a79ce481e8ab3347c92b3401f : SUCCESS in 1h 02m 45s","accounts_in_message":[],"_revision_number":5},{"id":"6b475c1f554c2dc1686c9c15aeb672a469c126c7","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-22 23:33:51.000000000","message":"Patch Set 5:\n\nBuild Failed \n\n* quobyteci:  cinder_quobyteci : FAILURE, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":5},{"id":"08cfd677d4fc61a6a1d1e7f4cd1fbdd065030420","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-22 23:45:12.000000000","message":"Patch Set 2:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4051/.","accounts_in_message":[],"_revision_number":2},{"id":"c153b81c05d573d1f0501dd3e080e170cbbd52cc","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-22 23:51:19.000000000","message":"Patch Set 2:\n\nBuild failed (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4227/.","accounts_in_message":[],"_revision_number":2},{"id":"28529135a4886717d845db68b4c3b998dd9bf38a","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 00:53:45.000000000","message":"Patch Set 3:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4052/.","accounts_in_message":[],"_revision_number":3},{"id":"d881f75b9243dae4da60c84cc2dfcd8b13460215","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-23 00:55:24.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/c555a6cb933e4155b3642361e59faf55\n\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/7aae391e92e14854b8e341eccfa1f5c0 : SUCCESS in 3m 58s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/6885b866bbef4e7fb56834c1f8556971 : FAILURE in 20m 18s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/9b19c25f201e47e4af0e012ac1696cf1 : FAILURE in 16m 05s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/9045054def7a4d648ba2104f6e4ba0a8 : FAILURE in 23m 39s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3fa36b281f394fe69d0ba4b5053c7a9e : FAILURE in 7m 32s\n- grenade https://zuul.opendev.org/t/openstack/build/bdd3880e74f64f8fad92a0327983864e : SUCCESS in 34m 26s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/9c8fd1b4b0234d16b9523b028743abf5 : SUCCESS in 45m 17s\n- tempest-integrated-storage https://zuul.opendev.org/t/openstack/build/de687a0f5b414bce902bd64674d523f1 : SUCCESS in 1h 34m 24s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/ab219442b2b9484fa9f6fb91a046ae98 : SUCCESS in 33m 09s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/df50c62a0d3345cfb8980f93d252f76f : SUCCESS in 6m 47s\n- cinder-code-coverage https://zuul.opendev.org/t/openstack/build/a6f0983571f943f7872206cc0d5e391d : FAILURE in 30m 44s (non-voting)\n- cinder-mypy https://zuul.opendev.org/t/openstack/build/c4a7da459b5340918e3764648d89406d : FAILURE in 3m 09s\n- cinder-tox-bandit-baseline https://zuul.opendev.org/t/openstack/build/873df2b8e7c6475c862a7165a959f48f : SUCCESS in 3m 30s (non-voting)\n- openstack-tox-functional-py311 https://zuul.opendev.org/t/openstack/build/c767c1e65dfa4a9d9a874a7b41ce4539 : FAILURE in 18m 28s\n- openstack-tox-functional-py313 https://zuul.opendev.org/t/openstack/build/9949e0884bba4ea7bb99ab59c86575fe : FAILURE in 18m 39s\n- openstack-tox-functional-py314 https://zuul.opendev.org/t/openstack/build/098af93df7444b0d86120892d33ebfe0 : FAILURE in 16m 59s (non-voting)\n- cinder-rally-task https://zuul.opendev.org/t/openstack/build/4bf9e2b91e414e2d976945b9c7b35cc5 : SUCCESS in 46m 48s (non-voting)\n- openstack-tox-pylint https://zuul.opendev.org/t/openstack/build/cdd6e6c74bd24f1bafe8885f75692fb8 : SUCCESS in 5m 49s (non-voting)\n- cinder-plugin-ceph-tempest https://zuul.opendev.org/t/openstack/build/5bce8b76e4574c74b236d9a7cc9b03d6 : FAILURE in 1h 49m 33s\n- cinder-plugin-ceph-tempest-mn-aa https://zuul.opendev.org/t/openstack/build/6fdc079f1a0f4094ae3c62f2f85bdcc5 : FAILURE in 2h 24m 23s (non-voting)\n- cinder-tempest-plugin-lvm-lio-barbican https://zuul.opendev.org/t/openstack/build/0a24bfa4327a4ca4803fe2d4e23b2287 : SUCCESS in 2h 19m 04s\n- cinder-tempest-plugin-lvm-lio-barbican-fips-py311 https://zuul.opendev.org/t/openstack/build/bd1211df507f4a8095a407cce6f92a79 : FAILURE in 1h 58m 53s (non-voting)\n- cinder-tempest-plugin-protection-functional https://zuul.opendev.org/t/openstack/build/844c01ee03804a91bdd2b987462e49bd : SUCCESS in 21m 56s\n- cinder-grenade-mn-sub-volbak https://zuul.opendev.org/t/openstack/build/9f5bdb116af04ff0b88229c4685a4b27 : SUCCESS in 54m 56s\n- cinder-tempest-lvm-multibackend https://zuul.opendev.org/t/openstack/build/45667a8ce7414fcdb80110c1141465fa : SUCCESS in 1h 34m 44s (non-voting)\n- cinder-for-glance-optimized https://zuul.opendev.org/t/openstack/build/286dab7d56b74cbf82dbdd5dfd8c12a3 : SUCCESS in 39m 18s (non-voting)\n- devstack-plugin-nfs-tempest-full https://zuul.opendev.org/t/openstack/build/539192ca33614eb390d1a98371df4e5e : SUCCESS in 1h 05m 40s\n- tempest-slow-py3 https://zuul.opendev.org/t/openstack/build/5d242b093bb042d59316e1c31b802f0b : FAILURE in 1h 40m 05s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/3bd78d3dbf2b44fca6f52eeb0d913f85 : SUCCESS in 59m 01s","accounts_in_message":[],"_revision_number":5},{"id":"f6a75e48ff8112f0071962ae4e7750bd274919a6","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 00:59:08.000000000","message":"Patch Set 3:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4228/.","accounts_in_message":[],"_revision_number":3},{"id":"795f0a9be1ca7fd22c9305e6a83c4c652e859b4e","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 01:59:54.000000000","message":"Patch Set 4:\n\nBuild failed (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4053/.","accounts_in_message":[],"_revision_number":4},{"id":"852c7c7e460ebd952e8cb0f71110db05e1b94a04","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 02:10:27.000000000","message":"Patch Set 4:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4229/.","accounts_in_message":[],"_revision_number":4},{"id":"9219ebeeb9bdd8e3a3a910456517e56d6d5bc13f","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 03:09:10.000000000","message":"Patch Set 5:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4054/.","accounts_in_message":[],"_revision_number":5},{"id":"9a8083cac4a19659026d521991e5f793b75cde52","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-23 03:29:32.000000000","message":"Patch Set 5:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4230/.","accounts_in_message":[],"_revision_number":5},{"id":"59390f2ed009fd7961b018300f56b921ccbe0a14","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-24 14:18:15.000000000","message":"Uploaded patch set 6.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":6},{"id":"2a8e527298e1001e3e698134c04058a704a1e948","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-24 14:47:36.000000000","message":"Patch Set 6:\n\nBuild failed.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/1b32da95d3064872adb42157c002be60 : FAILURE in 28m 52s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":6},{"id":"c0d250258b515cae632ef8da0dab17cab7fb1213","tag":"autogenerated:zuul:check","author":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"date":"2026-05-24 14:56:14.000000000","message":"Patch Set 6:\n\nBuild failed.\n\n- hpe-fc-driver build/13f5e9d82b934c2ba58cece92ecd51fe : FAILURE in 37m 34s\n\nWarning:\n  FC CI log: https://github.com/hpe-storage/hpe-ci-log/tree/master/logs/989836","accounts_in_message":[],"_revision_number":6},{"id":"73606583a2e3da98b2a801c7440438f3a5df4b54","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-24 15:03:42.000000000","message":"Patch Set 6:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/30f576bc79404469be6c9797e75b5c85\n\n- openstack-tox-py311-arm64 https://zuul.opendev.org/t/openstack/build/7777ebc10edf47f4bd3aeeacee1f75a8 : FAILURE in 23m 41s (non-voting)\n- openstack-tox-py313-arm64 https://zuul.opendev.org/t/openstack/build/7f4c5b764445476d8ec1763803b2ca50 : FAILURE in 23m 05s (non-voting)\n- openstack-tox-py314-arm64 https://zuul.opendev.org/t/openstack/build/a173cb29d1294da48f80c23928178d5d : FAILURE in 43m 41s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"5b936f50c10693839998a583d2ceacac6f994e6b","author":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"date":"2026-05-24 15:06:43.000000000","message":"Patch Set 6:\n\nBuild Succeeded: 3par-fc-driver: Logs: https://github.com/hpe-storage/hpe-3parfc-ci/tree/main/new_logs/989836 : SUCCESS","accounts_in_message":[],"_revision_number":6},{"id":"49c9c27febe7ce0a6d0a048b42ad16a134ad5892","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-24 15:13:36.000000000","message":"Patch Set 6:\n\nBuild Failed \n\n* quobyteci:  cinder_quobyteci : FAILURE, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":6},{"id":"91757743dcd07dceaf64bdb5e6d3602a9fb77fe8","author":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"date":"2026-05-24 15:13:46.000000000","message":"Patch Set 6:\n\nBuild succeeded.\n\n- cinder-storpool-tempest build/f9d98ff7c3224270a1dc023333f49022 : SUCCESS in 53m 05s\n- cinder-storpool-tempest-iscsi-multipath build/469750196cfa42faa1cdad6c75cd0ff5 : SUCCESS in 54m 12s","accounts_in_message":[],"_revision_number":6},{"id":"f6f1c977aabbeb904b65016e49431456333c5e8b","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-24 15:19:53.000000000","message":"Patch Set 6:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4233/.","accounts_in_message":[],"_revision_number":6},{"id":"68f8006bf827c7aa681a259776172142492b09bc","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-24 15:31:08.000000000","message":"Patch Set 6:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4057/.","accounts_in_message":[],"_revision_number":6},{"id":"77f93fa199dfae0a5e2afbbab4f37c0eeac7916d","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-24 16:33:46.000000000","message":"Uploaded patch set 7.","accounts_in_message":[],"_revision_number":7},{"id":"7d7be0c0f981dd2b439a080e547c3f8b433eb663","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-24 17:03:08.000000000","message":"Patch Set 7:\n\nBuild failed.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/5f045859600245e2a2cf141a741f5f17 : FAILURE in 28m 47s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":7},{"id":"29359d29a4069a1a4cf4264220c47bfcd0877b34","tag":"autogenerated:zuul:check","author":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"date":"2026-05-24 17:11:37.000000000","message":"Patch Set 7:\n\nBuild failed.\n\n- hpe-fc-driver build/3dbaeac2d4574499a11b82986af2d95c : FAILURE in 37m 23s\n\nWarning:\n  FC CI log: https://github.com/hpe-storage/hpe-ci-log/tree/master/logs/989836","accounts_in_message":[],"_revision_number":7},{"id":"d4a4d4aaa5f1b76b672afe987bc1cda183f2cbde","author":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"date":"2026-05-24 17:23:08.000000000","message":"Patch Set 7:\n\nBuild Succeeded: 3par-fc-driver: Logs: https://github.com/hpe-storage/hpe-3parfc-ci/tree/main/new_logs/989836 : SUCCESS","accounts_in_message":[],"_revision_number":7},{"id":"56ff13805b2c4ed3168f76aedfc3cc707a5c3c8c","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-24 17:31:52.000000000","message":"Patch Set 7:\n\nBuild Successful \n\n* quobyteci:  cinder_quobyteci : SUCCESS, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":7},{"id":"853cc4847e93d1c4f86e3916d170866fa74ae728","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-24 17:36:41.000000000","message":"Patch Set 7:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/c153c67ec40941db99579215b302cd28\n\n- openstack-tox-py311-arm64 https://zuul.opendev.org/t/openstack/build/dbc4d70b28c145ee96175588c72b0715 : FAILURE in 31m 43s (non-voting)\n- openstack-tox-py313-arm64 https://zuul.opendev.org/t/openstack/build/f441103219a14c5683ffe37e3d678c90 : TIMED_OUT in 1h 01m 01s (non-voting)\n- openstack-tox-py314-arm64 https://zuul.opendev.org/t/openstack/build/52dde994611c431ba47110bd2a371d4b : FAILURE in 30m 20s (non-voting)","accounts_in_message":[],"_revision_number":7},{"id":"31297956084374603e280bcfa6d036f0cbdde918","author":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"date":"2026-05-24 17:37:48.000000000","message":"Patch Set 7:\n\nBuild failed.\n\n- cinder-storpool-tempest build/75d6ea26f9564403963aacfe48b6d27d : SUCCESS in 51m 12s\n- cinder-storpool-tempest-iscsi-multipath build/c4217ca1d09645a9b5db2c26d4800ccc : FAILURE in 1h 02m 39s","accounts_in_message":[],"_revision_number":7},{"id":"1ef96f3bb04d1642d638123f807d86c914f3a26a","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-24 17:39:37.000000000","message":"Patch Set 7:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4058/.","accounts_in_message":[],"_revision_number":7},{"id":"c80b2f03800d1f0af52424c73c72862b5cc06500","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-24 17:42:37.000000000","message":"Patch Set 7:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4234/.","accounts_in_message":[],"_revision_number":7},{"id":"ea29fdfe4f1dd57ebed9cee8eaae94682a551759","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-24 20:16:20.000000000","message":"Patch Set 7: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/2fea520a77d34c81a48fd8411f6d2486\n\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d7d18b9905af409e9bbbe7a225532bc0 : SUCCESS in 4m 01s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/3c0dee959d8c41cd9dba2be92c0031e7 : FAILURE in 20m 45s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/ebc19709da2d492f90523c4944bba590 : FAILURE in 18m 43s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/49029ad3771549a0ab2f3ca0e9277a7a : FAILURE in 22m 57s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f612ff2c828e4893a7245508c90b69ab : SUCCESS in 13m 56s\n- grenade https://zuul.opendev.org/t/openstack/build/7cc8b0f762d94a6d8824992970fea968 : SUCCESS in 1h 09m 20s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/f1ba4d976ec7418498942b7f7a539225 : SUCCESS in 1h 11m 02s\n- tempest-integrated-storage https://zuul.opendev.org/t/openstack/build/6ce50c11a1514ef98339d399d84d4d3e : SUCCESS in 1h 48m 38s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/b779375edc8540fd8eb91f98115348dc : SUCCESS in 1h 00m 26s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/ba9b79db787644cf8982721a6a02ea55 : SUCCESS in 8m 42s\n- cinder-code-coverage https://zuul.opendev.org/t/openstack/build/7a0d02d1160e42d5a1f7520bca2fd47b : FAILURE in 28m 32s (non-voting)\n- cinder-mypy https://zuul.opendev.org/t/openstack/build/5751d414af144776b4afdf9b47b1ce63 : SUCCESS in 3m 06s\n- cinder-tox-bandit-baseline https://zuul.opendev.org/t/openstack/build/9ff551b366e245a88a7768ded7e3ae1d : SUCCESS in 6m 12s (non-voting)\n- openstack-tox-functional-py311 https://zuul.opendev.org/t/openstack/build/729bd6cf17c74d2c8da0395db0021c76 : FAILURE in 12m 52s\n- openstack-tox-functional-py313 https://zuul.opendev.org/t/openstack/build/5c76dbee3477471ab095dc460c92cf97 : FAILURE in 11m 11s\n- openstack-tox-functional-py314 https://zuul.opendev.org/t/openstack/build/4b74bebae586459b8b2bf702ce68bdac : SUCCESS in 12m 23s (non-voting)\n- cinder-rally-task https://zuul.opendev.org/t/openstack/build/8472244bc9c8497f9d9ca4fd98cb1e3c : SUCCESS in 39m 14s (non-voting)\n- openstack-tox-pylint https://zuul.opendev.org/t/openstack/build/68642345e50647a499fd83bf538632a1 : SUCCESS in 6m 00s (non-voting)\n- cinder-plugin-ceph-tempest https://zuul.opendev.org/t/openstack/build/febf5d5eba724df2b495d4f109124096 : FAILURE in 1h 36m 10s\n- cinder-plugin-ceph-tempest-mn-aa https://zuul.opendev.org/t/openstack/build/a9601daacb3d4c3b9cdc02e2950580c7 : TIMED_OUT Host unreachable in 3h 36m 27s (non-voting)\n- cinder-tempest-plugin-lvm-lio-barbican https://zuul.opendev.org/t/openstack/build/ea78c89d0b23458c9a03ce4ea31b06e9 : FAILURE in 2h 18m 41s\n- cinder-tempest-plugin-lvm-lio-barbican-fips-py311 https://zuul.opendev.org/t/openstack/build/fdf2fbf2d8454ca3add88df3eee32e5e : FAILURE in 1h 51m 33s (non-voting)\n- cinder-tempest-plugin-protection-functional https://zuul.opendev.org/t/openstack/build/cadcb1c5bc66492885be84da87d5962f : SUCCESS in 20m 54s\n- cinder-grenade-mn-sub-volbak https://zuul.opendev.org/t/openstack/build/73654f71757a469a989368b6743ecf69 : SUCCESS in 1h 14m 33s\n- cinder-tempest-lvm-multibackend https://zuul.opendev.org/t/openstack/build/bba940ed41444af6a668eea64ca128f9 : SUCCESS in 1h 36m 51s (non-voting)\n- cinder-for-glance-optimized https://zuul.opendev.org/t/openstack/build/b309eb02b9c342a388ff42a72fd6368b : SUCCESS in 46m 37s (non-voting)\n- devstack-plugin-nfs-tempest-full https://zuul.opendev.org/t/openstack/build/510715e4036147958ba0a631b4b2fee4 : SUCCESS in 2h 11m 09s\n- tempest-slow-py3 https://zuul.opendev.org/t/openstack/build/20836db23d5b41eaa016f6adbbd09205 : SUCCESS in 53m 47s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/a9877152368140b787341b2cfdaaa434 : SUCCESS in 1h 04m 56s","accounts_in_message":[],"_revision_number":7},{"id":"9b9bce333a765a3b7e40b1ea101eea18fb2a63a1","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-26 15:29:01.000000000","message":"Uploaded patch set 8.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":8},{"id":"a28c8d6dc208487915c2b2a58f342538b8f9128a","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-26 16:28:34.000000000","message":"Patch Set 8:\n\nBuild Failed \n\n* quobyteci:  cinder_quobyteci : FAILURE, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":8},{"id":"e260584141a28d5acc1c96479556bd8a488b7d2b","tag":"autogenerated:zuul:check","author":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"date":"2026-05-26 17:34:59.000000000","message":"Patch Set 8:\n\nBuild succeeded.\n\n- hpe-fc-driver build/04b40f0b859449fcb0c725efc3ef9646 : SUCCESS in 35m 37s\n\nWarning:\n  FC CI log: https://github.com/hpe-storage/hpe-ci-log/tree/master/logs/989836","accounts_in_message":[],"_revision_number":8},{"id":"8cd14b29fa7ec4acec24483f5740b6402b56a726","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-26 18:21:25.000000000","message":"Patch Set 8:\n\nBuild failed.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/4deb9f030fff474db13c3bdb48352403 : FAILURE in 40m 40s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":8},{"id":"880586b08b89cc17be1ba4e0fc0f28bfce3da9da","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-26 18:53:07.000000000","message":"Patch Set 8:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/161b03bbf3224a19ad496ef7cdfd11e1\n\n- openstack-tox-py311-arm64 https://zuul.opendev.org/t/openstack/build/0beba92ec39e43f0bde592c3f497eb4a : FAILURE in 28m 27s (non-voting)\n- openstack-tox-py313-arm64 https://zuul.opendev.org/t/openstack/build/7a53373b0fef4acf8c2d3f8ff5c137b5 : FAILURE in 22m 04s (non-voting)\n- openstack-tox-py314-arm64 https://zuul.opendev.org/t/openstack/build/910a05aa174c41e7919642085216c1a2 : FAILURE in 42m 51s (non-voting)","accounts_in_message":[],"_revision_number":8},{"id":"ac8c4cbb5b3da4e93928d9d0b0f9f7aa977ddbbc","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-26 19:20:34.000000000","message":"Patch Set 8: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/973330918d0746ca8284e6ce8e81bda6\n\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/28b30fdf80594faa8eb65c7d5c975eca : SUCCESS in 4m 13s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/e90f3c113fbb45eea63d7d5ef9ca8ada : SUCCESS in 12m 14s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/11dbac5f22364e688cd26383fcf60294 : FAILURE in 11m 37s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/8410cdfaa8c94a3e97fb4f6876662c48 : SUCCESS in 10m 27s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/230de9ee92fe402095c79fb1838869bc : SUCCESS in 14m 30s\n- grenade https://zuul.opendev.org/t/openstack/build/0a1a932888fb41fa96c6c7208b654fa4 : SUCCESS in 1h 05m 23s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/719f3671263d41ebb11bde1229d2fa44 : SUCCESS in 1h 20m 33s\n- tempest-integrated-storage https://zuul.opendev.org/t/openstack/build/0e35778664f44a9c8771411763a6e499 : SUCCESS in 1h 22m 45s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/3b56ef2bcad745218f6bb95d8e725f27 : SUCCESS in 1h 09m 05s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/cfecf7f3365b43f19520d70778468859 : SUCCESS in 9m 40s\n- cinder-code-coverage https://zuul.opendev.org/t/openstack/build/cb440ecd00764918b53fb30176f9aece : FAILURE in 32m 47s (non-voting)\n- cinder-mypy https://zuul.opendev.org/t/openstack/build/6510bba5b8024ed0b3bf3a37bb43173a : SUCCESS in 5m 03s\n- cinder-tox-bandit-baseline https://zuul.opendev.org/t/openstack/build/b106d798fb4c4192af39ce9238cf4551 : SUCCESS in 6m 43s (non-voting)\n- openstack-tox-functional-py311 https://zuul.opendev.org/t/openstack/build/7068d116b541430aa1d6815979c2cde0 : FAILURE in 21m 26s\n- openstack-tox-functional-py313 https://zuul.opendev.org/t/openstack/build/73d1f934809b45fbb32207e20bb87044 : FAILURE in 17m 37s\n- openstack-tox-functional-py314 https://zuul.opendev.org/t/openstack/build/5c2fa384f81a412082133a50ac53e970 : FAILURE in 17m 13s (non-voting)\n- cinder-rally-task https://zuul.opendev.org/t/openstack/build/859def137e4d4827ab4af422f9fb4445 : SUCCESS in 44m 12s (non-voting)\n- openstack-tox-pylint https://zuul.opendev.org/t/openstack/build/f41490f1ea424a1d9f3c4a65a20cb017 : SUCCESS in 3m 46s (non-voting)\n- cinder-plugin-ceph-tempest https://zuul.opendev.org/t/openstack/build/4be3c43ea1f648f2b7dd06e8146cfe73 : FAILURE in 1h 50m 13s\n- cinder-plugin-ceph-tempest-mn-aa https://zuul.opendev.org/t/openstack/build/8320fbf8aa9f487ab15171291ca60043 : TIMED_OUT Host unreachable in 3h 42m 48s (non-voting)\n- cinder-tempest-plugin-lvm-lio-barbican https://zuul.opendev.org/t/openstack/build/bf1e065426304ac9862efd4991de6f44 : SUCCESS in 1h 59m 01s\n- cinder-tempest-plugin-lvm-lio-barbican-fips-py311 https://zuul.opendev.org/t/openstack/build/ec504fd5b16e4acfa1d2732e14df3669 : FAILURE in 1h 50m 47s (non-voting)\n- cinder-tempest-plugin-protection-functional https://zuul.opendev.org/t/openstack/build/379b90f87bfc4ae382cca1d5454ace4c : SUCCESS in 22m 21s\n- cinder-grenade-mn-sub-volbak https://zuul.opendev.org/t/openstack/build/dc0331fa53f24c35820a6a461af738a0 : SUCCESS in 1h 18m 46s\n- cinder-tempest-lvm-multibackend https://zuul.opendev.org/t/openstack/build/4ef76e8065554560b843926a6fb79275 : SUCCESS in 1h 46m 19s (non-voting)\n- cinder-for-glance-optimized https://zuul.opendev.org/t/openstack/build/511a0b3ecdea425dbceeaf37c3c3ee85 : SUCCESS in 25m 23s (non-voting)\n- devstack-plugin-nfs-tempest-full https://zuul.opendev.org/t/openstack/build/9b6f64d673b84ee1846847bc9aa7cebf : FAILURE in 1h 08m 04s\n- tempest-slow-py3 https://zuul.opendev.org/t/openstack/build/49a5c85dd482448397f22ad1c5ad0036 : SUCCESS in 1h 46m 38s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/bd01a466856742d79ce93839760091a3 : SUCCESS in 1h 01m 29s","accounts_in_message":[],"_revision_number":8},{"id":"9c7bcedbdae4262048667f94ece95648a5bd9652","author":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"date":"2026-05-26 19:56:31.000000000","message":"Patch Set 8:\n\nBuild failed.\n\n- cinder-storpool-tempest build/4f678e75922a4918bb78ab562be80587 : FAILURE in 58m 39s\n- cinder-storpool-tempest-iscsi-multipath build/6a969a217c87499cb5691370f626063b : SUCCESS in 56m 17s","accounts_in_message":[],"_revision_number":8},{"id":"dd4f487ea8b3a565814ab70c6c96e3a3b2bcb9da","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-26 20:43:52.000000000","message":"Patch Set 8:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4068/.","accounts_in_message":[],"_revision_number":8},{"id":"4c984eea8f1774a038e8846667fee192685b11c7","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-26 21:39:49.000000000","message":"Patch Set 8:\n\nBuild failed (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4244/.","accounts_in_message":[],"_revision_number":8},{"id":"4714ef9add7244d3731f66a08684717a0b6ad8bb","author":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"date":"2026-05-27 13:01:11.000000000","message":"Patch Set 8:\n\nBuild Failed: 3par-fc-driver: Logs: https://github.com/hpe-storage/hpe-3parfc-ci/tree/main/new_logs/989836 : FAILURE","accounts_in_message":[],"_revision_number":8},{"id":"b8f9bd612b83b4f18abd18cfd2b97ee2a5bb2c49","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"date":"2026-05-27 20:59:46.000000000","message":"Uploaded patch set 9.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":9},{"id":"2fa9926ac97429cbabded8c21f2ecba8cdf4b53a","tag":"autogenerated:zuul:check","author":{"_account_id":39096,"name":"HPE AlletraMP FC CI","display_name":"HPE AlletraMP FC CI","email":"diganta.bora@hpe.com","username":"digantabora"},"date":"2026-05-27 21:37:48.000000000","message":"Patch Set 9:\n\nBuild succeeded.\n\n- hpe-fc-driver build/4add9cd334634124a301ea69dfdcb0a3 : SUCCESS in 37m 30s\n\nWarning:\n  FC CI log: https://github.com/hpe-storage/hpe-ci-log/tree/master/logs/989836","accounts_in_message":[],"_revision_number":9},{"id":"5323fde4f069b90719713d8bc39cee6fc9d6aaab","author":{"_account_id":22248,"name":"HPE STORAGE CI","display_name":"HPE NIMBLE STORAGE CI","email":"hp.cinder.blr.ci@groups.ext.hpe.com","username":"hp-storage-blr-ci","tags":["SERVICE_USER"]},"date":"2026-05-27 21:40:04.000000000","message":"Patch Set 9:\n\nBuild succeeded.\n\n- 3par-iscsi-driver-master-client-pip-ssa02-dsvm build/4d9a5ff434d043f5ba76a49b561e74f7 : SUCCESS in 39m 46s\n\nWarning:\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836\n  iSCSI CI log: https://github.com/hpe-storage/hpe_cinder_logs/blob/master/new_logs/989836","accounts_in_message":[],"_revision_number":9},{"id":"1be777e8938622c67e70bf10feab370417447da8","tag":"autogenerated:zuul:check","author":{"_account_id":38252,"name":"HPE AlletraMP NVMeTCP CI","display_name":"HPE AlletraMP NVMeTCP CI","email":"yogeshwari.gitte-ext@hpe.com","username":"yogeshwarigitte"},"date":"2026-05-27 21:45:50.000000000","message":"Patch Set 9:\n\nBuild succeeded.\n\n- hpe-nvmetcp-driver build/91846d293135413f979060d25f4d260c : SUCCESS in 45m 32s\n\nWarning:\n  NVMe CI log: https://github.com/hpe-storage/hpe-nvmetcp-ci-logs/tree/master/new_logs/989836","accounts_in_message":[],"_revision_number":9},{"id":"4355dc0d7a5f1017e5fdd0b8f91a1ad72abb7247","author":{"_account_id":14384,"name":"Quobyte CI","email":"openstack-ci-external@quobyte.com","username":"quobyteci","tags":["SERVICE_USER"]},"date":"2026-05-27 22:07:05.000000000","message":"Patch Set 9:\n\nBuild Failed \n\n* quobyteci:  cinder_quobyteci : FAILURE, logs at: https://oscilogs.quobyte.com/logs/Icdd28affc73fd34491b656a68410dce8e46264d4 .","accounts_in_message":[],"_revision_number":9},{"id":"5bce56d5c41d97dfc5da0d26e406cb8cb933f2e2","author":{"_account_id":15670,"name":"StorPool OpenStack CI","email":"openstack-ci@storpool.com","username":"storpoolci","tags":["SERVICE_USER"]},"date":"2026-05-27 23:33:15.000000000","message":"Patch Set 9:\n\nBuild failed.\n\n- cinder-storpool-tempest build/412f569f0b014c3fa9cb648aa13cd3b8 : FAILURE in 59m 48s\n- cinder-storpool-tempest-iscsi-multipath build/f85ee68fa9944f04a0a240b26798683e : SUCCESS in 1h 01m 15s","accounts_in_message":[],"_revision_number":9},{"id":"74f2c5efc52f41b238f9349445fe4e5943823bef","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-27 23:59:31.000000000","message":"Patch Set 9: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/88681c9b0f7642438b2ae674e9c2a91d\n\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2eb0ee3aae474d548ef76b6364669aa1 : SUCCESS in 6m 16s\n- openstack-tox-py311 https://zuul.opendev.org/t/openstack/build/c94922a240584ce7a35d8a9ce08c4489 : SUCCESS in 10m 16s\n- openstack-tox-py313 https://zuul.opendev.org/t/openstack/build/d71c021609354c78a493fde60e7d696f : SUCCESS in 9m 31s\n- openstack-tox-py314 https://zuul.opendev.org/t/openstack/build/04cdc5b78ec34373ad953d2843b99164 : SUCCESS in 14m 56s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/fe943dcd57344dfb90cff56def712a9b : SUCCESS in 7m 15s\n- grenade https://zuul.opendev.org/t/openstack/build/6f1bd6dd77274a6fbeac3e26998ee7e1 : SUCCESS in 46m 58s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/2f56fecc622a4f03b3ea1087f93a54df : SUCCESS in 39m 57s\n- tempest-integrated-storage https://zuul.opendev.org/t/openstack/build/6732aead817246a983bb75e8f188e00d : FAILURE in 1h 18m 05s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/30ee2ae0164545448a810710d9548b06 : SUCCESS in 1h 03m 54s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/054fa74922c64f0c92929b5bea34e96d : SUCCESS in 5m 13s\n- cinder-code-coverage https://zuul.opendev.org/t/openstack/build/fa0e3126272a4147b5e6190294362cf5 : SUCCESS in 23m 38s (non-voting)\n- cinder-mypy https://zuul.opendev.org/t/openstack/build/7e6dce84d84b4a02b485b544a5f36b70 : SUCCESS in 4m 53s\n- cinder-tox-bandit-baseline https://zuul.opendev.org/t/openstack/build/f34b9eedc3bb461d8c8638e8626c51a3 : SUCCESS in 5m 34s (non-voting)\n- openstack-tox-functional-py311 https://zuul.opendev.org/t/openstack/build/34758f26265a40d591431cecd2328962 : SUCCESS in 9m 25s\n- openstack-tox-functional-py313 https://zuul.opendev.org/t/openstack/build/895d2b5b62e94f70a28a6eea36e1af6e : SUCCESS in 5m 16s\n- openstack-tox-functional-py314 https://zuul.opendev.org/t/openstack/build/48a5c7b28ce24c30b76dce0f420f48d9 : SUCCESS in 7m 58s (non-voting)\n- cinder-rally-task https://zuul.opendev.org/t/openstack/build/f8ca10a3f9fb4bdeb14139783f584fa1 : SUCCESS in 41m 35s (non-voting)\n- openstack-tox-pylint https://zuul.opendev.org/t/openstack/build/d099851f7da140ddac0c02b9db447b3c : SUCCESS in 6m 51s (non-voting)\n- cinder-plugin-ceph-tempest https://zuul.opendev.org/t/openstack/build/99cd0566d9e74c17b8d1663cdf5b5a9c : FAILURE in 1h 10m 34s\n- cinder-plugin-ceph-tempest-mn-aa https://zuul.opendev.org/t/openstack/build/16d6817688c3400eadf689ee6c82c716 : FAILURE in 2h 49m 58s (non-voting)\n- cinder-tempest-plugin-lvm-lio-barbican https://zuul.opendev.org/t/openstack/build/c4673939d84447ca971eb696af857710 : SUCCESS in 1h 27m 57s\n- cinder-tempest-plugin-lvm-lio-barbican-fips-py311 https://zuul.opendev.org/t/openstack/build/85e885a5acb3445a92e4a4a051f04f47 : FAILURE in 1h 59m 31s (non-voting)\n- cinder-tempest-plugin-protection-functional https://zuul.opendev.org/t/openstack/build/6825af5ac9234f89973084b2f07aa883 : SUCCESS in 18m 48s\n- cinder-grenade-mn-sub-volbak https://zuul.opendev.org/t/openstack/build/db2d357a136443f88d610a5c221ac2bd : SUCCESS in 54m 38s\n- cinder-tempest-lvm-multibackend https://zuul.opendev.org/t/openstack/build/2d853ec9cdcc43b9826801148938b10c : SUCCESS in 1h 38m 03s (non-voting)\n- cinder-for-glance-optimized https://zuul.opendev.org/t/openstack/build/d124ad1242614acd99f6fac260a373f5 : SUCCESS in 34m 37s (non-voting)\n- devstack-plugin-nfs-tempest-full https://zuul.opendev.org/t/openstack/build/5cf78c3760f540f1b997e055e1d27135 : SUCCESS in 2h 03m 52s\n- tempest-slow-py3 https://zuul.opendev.org/t/openstack/build/3fad5cc2d7e34f778faf3412dc4bfab8 : SUCCESS in 52m 23s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/5c6105ca6f194f36a608fe17851e4a07 : SUCCESS in 59m 32s","accounts_in_message":[],"_revision_number":9},{"id":"cf9e531f3e92056188b66488da7483a2b69f15e0","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-28 00:21:28.000000000","message":"Patch Set 9:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/0a0f9e3e1b994e8e949ab4de2466b60c\n\n- openstack-tox-py311-arm64 https://zuul.opendev.org/t/openstack/build/9fe89c7706b64203ad74cdb2946637a2 : FAILURE in 19m 24s (non-voting)\n- openstack-tox-py313-arm64 https://zuul.opendev.org/t/openstack/build/5be19985238242ca9cb4e46b2070a013 : SUCCESS in 21m 44s (non-voting)\n- openstack-tox-py314-arm64 https://zuul.opendev.org/t/openstack/build/b54773c444784ed6b9f8b10fcd0b054d : SUCCESS in 15m 00s (non-voting)","accounts_in_message":[],"_revision_number":9},{"id":"0b71e6145661912aff6c4ad1762dff22e1446d7c","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-28 05:09:50.000000000","message":"Patch Set 9:\n\nBuild succeeded (iSCSI).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-iscsi/4264/.","accounts_in_message":[],"_revision_number":9},{"id":"278f9a21d3aaff37e3d2ca0a1046b75b6802f2f3","author":{"_account_id":33746,"name":"YADRO TATLIN CI","email":"tatlin_ci@yadro.com","username":"tatlin-ci"},"date":"2026-05-28 06:25:39.000000000","message":"Patch Set 9:\n\nBuild succeeded (FC).\nLogs: http://tempest-openstack.spb.yadro.com/openstack-gerrit-trigger-fc/4088/.","accounts_in_message":[],"_revision_number":9},{"id":"68436540008cbea97baa93bb0a4a9841581d0c6c","author":{"_account_id":38770,"name":"HPE 3PAR Storage FC CI","display_name":"HPE 3PAR Storage FC CI","email":"shubhangip@hpe.com","username":"arcusci"},"date":"2026-05-28 12:41:56.000000000","message":"Patch Set 9:\n\nBuild Failed: 3par-fc-driver: Logs: https://github.com/hpe-storage/hpe-3parfc-ci/tree/main/new_logs/989836 : FAILURE","accounts_in_message":[],"_revision_number":9},{"id":"467ff34077ae8cb77406522f275f9428171c75b2","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 19:51:51.000000000","message":"Patch Set 5:\n\nBuild Aborted \n\nhttp://10.136.74.243:8080/job/vmstore-cinder-ci-gerrit/1953/ : ABORTED","accounts_in_message":[],"_revision_number":5},{"id":"e3e029bfc066b91573a5757bc49449b5d992288d","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:01:44.000000000","message":"Patch Set 1:\n\nBuild Failed","accounts_in_message":[],"_revision_number":1},{"id":"2b6af06b77321e1cd2f9cab9f3f21ceb0c1f65e6","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:01:45.000000000","message":"Patch Set 8:\n\nBuild Failed","accounts_in_message":[],"_revision_number":8},{"id":"aec66b90585f9b8985803bdaabdd119b9e4206cf","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:03:00.000000000","message":"Patch Set 6:\n\nBuild Failed","accounts_in_message":[],"_revision_number":6},{"id":"4d63daab3bf595c88b64f3343215e7dbd8f309b8","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:04:15.000000000","message":"Patch Set 2:\n\nBuild Failed","accounts_in_message":[],"_revision_number":2},{"id":"2d7a9413ef96d2862fd71c7ebc5f05f895c0a153","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:04:36.000000000","message":"Patch Set 9:\n\nBuild Failed","accounts_in_message":[],"_revision_number":9},{"id":"68f0fb3c8e1de1a6f1b0823b58a17bf31dd2b413","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:04:41.000000000","message":"Patch Set 3:\n\nBuild Failed","accounts_in_message":[],"_revision_number":3},{"id":"3002b0fcfd31f7cc55b332c17f9676567f037029","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:05:27.000000000","message":"Patch Set 7:\n\nBuild Failed","accounts_in_message":[],"_revision_number":7},{"id":"7505e3fdd3db75873508fe5d1b6bc19208675325","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":38702,"name":"Ranjith","display_name":"Ranjith","email":"goldranjith@gmail.com","username":"goldranjith"},"date":"2026-06-19 20:06:07.000000000","message":"Patch Set 4:\n\nBuild Failed","accounts_in_message":[],"_revision_number":4}],"current_revision_number":9,"current_revision":"d9afe47bfb56e3605a68f7c468fa82dacda71d93","revisions":{"363aa932c1e6982d01d0f7470960525e0a22ad8a":{"kind":"REWORK","_number":1,"created":"2026-05-22 21:29:51.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/1"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-04-23 22:45:28.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 21:29:16.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Verified empirically: Connection.stop_consuming() is never called\n  in cinder\u0027s shutdown path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nTested operations surviving pod termination (qa-de-1):\n\ntest_nova_cinder_interactions.py — 7/7 PASS:\n  T1-T4 (attach with ConnectorRejected + kill scenarios)\n  T9 (operator migrate baseline)\n  T10 (kill source mid-migrate — canonical SIG_IGN reproducer;\n       migrate_volume returned 39.8s post-SIGTERM)\n  T11 (kill dest mid-migrate)\n\ntest_graceful_shutdown.py — single-op tests 6/6 PASS on KVM/NetApp:\n  volume create, delete, extend, clone, snapshot create, snapshot delete\n\ntest_graceful_shutdown.py — concurrency suite 5/6 PASS on VMware/FCD:\n  sameop_create_x3, sameop_extend_x3, sameop_clone_x3\n  mixed_create_extend_clone, mixed_create_extend_snapshot_delete\n  (1 failure: vCenter InvalidArgument env flake, not a drain regression)\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/363aa932c1e6982d01d0f7470960525e0a22ad8a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/363aa932c1e6982d01d0f7470960525e0a22ad8a"}]},"branch":"refs/heads/master"},"3837918647a9b265c62e42497f923a190bf9e9a2":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2026-05-22 21:32:03.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/2"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-04-23 22:45:28.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 21:31:46.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/3837918647a9b265c62e42497f923a190bf9e9a2"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/3837918647a9b265c62e42497f923a190bf9e9a2"}]},"branch":"refs/heads/master"},"6be4ae94c7adecfc2583ffedb0aaf104f073091e":{"kind":"REWORK","_number":3,"created":"2026-05-22 21:36:57.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/3"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-04-23 22:45:28.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 21:35:01.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/6be4ae94c7adecfc2583ffedb0aaf104f073091e"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/6be4ae94c7adecfc2583ffedb0aaf104f073091e"}]},"branch":"refs/heads/master"},"687ab49b25eec8c036c5e34cf35e761d4c215c00":{"kind":"REWORK","_number":4,"created":"2026-05-22 22:16:22.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/4"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-04-23 22:45:28.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:14:40.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/687ab49b25eec8c036c5e34cf35e761d4c215c00"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/687ab49b25eec8c036c5e34cf35e761d4c215c00"}]},"branch":"refs/heads/master"},"a06164bc6c7dc9f8c052e0154ce58e88ee8577b8":{"kind":"REWORK","_number":5,"created":"2026-05-22 22:28:11.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/5"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/a06164bc6c7dc9f8c052e0154ce58e88ee8577b8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/a06164bc6c7dc9f8c052e0154ce58e88ee8577b8"}]},"branch":"refs/heads/master"},"3d16fe71be04f6c94c33307e4f0965e93266c62d":{"kind":"REWORK","_number":6,"created":"2026-05-24 14:18:15.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/6"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-24 14:18:05.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/3d16fe71be04f6c94c33307e4f0965e93266c62d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/3d16fe71be04f6c94c33307e4f0965e93266c62d"}]},"branch":"refs/heads/master"},"922f57140ddfc6a2f2373ce32bad7df032606843":{"kind":"REWORK","_number":7,"created":"2026-05-24 16:33:46.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/7"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-24 16:33:42.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/922f57140ddfc6a2f2373ce32bad7df032606843"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/922f57140ddfc6a2f2373ce32bad7df032606843"}]},"branch":"refs/heads/master"},"b97547fabe599cf8efe51b1f6cdceb1fc7b00457":{"kind":"REWORK","_number":8,"created":"2026-05-26 15:29:01.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/8"}}},"commit":{"parents":[{"commit":"849ddb6cef2b06084b7b116cb2ae5ce7da610e1b","subject":"Merge \"Use futurist-based thread pool for volume manager\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/849ddb6cef2b06084b7b116cb2ae5ce7da610e1b"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-26 15:28:39.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nThe fix: install SIG_IGN for SIGTERM/SIGINT/SIGHUP at the start of\nService.stop(). cinder-volume runs as N forked child processes (one per\nbackend) under oslo_service.ProcessLauncher. On SIGTERM, oslo_service\u0027s\nchild handler calls SignalHandler.clear() which resets all handlers to\nSIG_DFL. The parent then sends a second SIGTERM via os.kill(child_pid,\nSIGTERM). Without SIG_IGN, the child terminates immediately — even\nthough it\u0027s still inside pool.waitall() waiting for in-flight RPC\nhandlers to finish.\n\nHow it works:\n\nPhase 1: Skip rpcserver.stop(). We do NOT call rpcserver.stop() because\n  it triggers an eventlet socket race (\"simultaneous read on fileno\")\n  that disrupts outbound HTTP/RPC connections used by in-flight ops.\n  Broker-side consumer deregistration happens automatically when the\n  AMQP channel closes at process exit. During the gap between SIGTERM\n  and exit, two things protect against message loss:\n  - Heartbeat continues during drain, keeping the service \"up\" in the\n    DB. The scheduler only stops routing new work once service_down_time\n    expires AFTER the process actually exits.\n  - @reject_if_draining unconditionally rejects any late RPC that does\n    arrive, raising ServiceUnavailable so the caller retries on a\n    healthy instance.\n\nPhase 2: Block in pool.waitall() until all in-flight RPC handler\n  greenthreads complete their operations. The pool is the rpcserver\u0027s\n  _work_executor._pool GreenPool — we access it directly since we\n  skipped rpcserver.stop().\n\nPhase 3: Stop coordination, call super().stop(), cleanup threadpool\n  executor. Process exits cleanly.\n\nSupporting mechanisms:\n- Worker entry heartbeat (set_workers decorator in cleanable.py):\n  touches worker DB entries every 10s during operations, preventing\n  new pod init_host do_cleanup from resetting in-flight volumes to\n  error.\n- do_cleanup freshness check (manager.py): skips worker entries updated\n  within service_down_time — only cleans up truly stale entries from\n  crashed processes, not entries being actively heartbeated by a\n  draining pod.\n- Backup restore heartbeat (backup/manager.py): touches\n  backup.updated_at every 10s during restore, preventing new backup\n  pod init_host from triggering BackupRestoreCancel.\n- Backup _cleanup_one_backup freshness check: skips backups in\n  creating/restoring state if updated_at is recent.\n- Backup _detach_device no-reraise: if detach fails during shutdown\n  (e.g. RPC timeout), log the error but do not reraise. Data integrity\n  is preserved; dangling export is cleaned on next startup.\n- reject_if_draining decorator (volume/manager.py): unconditionally\n  rejects new RPC calls on a draining service.\n- CreateVolumeOnFinishTask unconditional write: ensures the final\n  status\u003d\u0027available\u0027 write overwrites any transient \u0027error\u0027 that the\n  new pod\u0027s cleanup may have set during the overlap window.\n\nKey design decisions:\n- Heartbeat is NOT suppressed during drain. This keeps the service\n  marked \"up\" in the DB, preventing the scheduler from prematurely\n  routing work away and preventing init_host cleanup on the new pod\n  from firing too early. The service only goes \"down\" after the\n  process exits and misses service_down_time heartbeat cycles.\n- rpcserver.stop() is NOT called. Calling it triggers\n  AMQPListener.stop() -\u003e conn.stop_consuming() which causes the\n  eventlet socket race. Instead, we let the process exit naturally\n  and rely on the OS closing the AMQP socket.\n- The oslo.messaging companion patch (review.opendev.org 986243) that\n  adds basic.cancel to Connection.stop_consuming() is NOT required.\n  Connection.stop_consuming() is never called in cinder\u0027s shutdown\n  path because we skip rpcserver.stop().\n\nRequires (separate changes):\n- dumb-init --single-child on container commands (ensures\n  ProcessLauncher parent waits for all children before exit)\n- terminationGracePeriodSeconds \u003e\u003d graceful_shutdown_timeout on pod spec\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/b97547fabe599cf8efe51b1f6cdceb1fc7b00457"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/b97547fabe599cf8efe51b1f6cdceb1fc7b00457"}]},"branch":"refs/heads/master"},"d9afe47bfb56e3605a68f7c468fa82dacda71d93":{"kind":"REWORK","_number":9,"created":"2026-05-27 20:59:46.000000000","uploader":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"ref":"refs/changes/36/989836/9","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/cinder","ref":"refs/changes/36/989836/9","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/cinder refs/changes/36/989836/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/cinder refs/changes/36/989836/9"}}},"commit":{"parents":[{"commit":"212e7e15b15f626b50489e1595cc5fc1d6d51d1f","subject":"Remove unused type ignore for requests.packages.urllib3","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/212e7e15b15f626b50489e1595cc5fc1d6d51d1f"}]}],"author":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-22 22:27:40.000000000","tz":-240},"committer":{"name":"Walter Boring","email":"waboring@hemna.com","date":"2026-05-27 20:59:44.000000000","tz":-240},"subject":"Implement graceful shutdown for Cinder services","message":"Implement graceful shutdown for Cinder services\n\nThree-phase graceful shutdown that allows in-flight volume and backup\noperations to complete before the pod exits during Kubernetes rolling\nupdates. Covers both cinder-volume and cinder-backup services.\n\nPhase 1: Skip consumer cancel (avoid eventlet socket race).\nPhase 2: Block in pool.waitall() for in-flight RPC handlers.\nPhase 3: Stop coordination, cleanup threadpool. Process exits.\n\nSupporting mechanisms:\n- SIG_IGN for SIGTERM/SIGINT/SIGHUP (prevent premature child kill)\n- Worker entry heartbeat in set_workers decorator (interruptible)\n- do_cleanup freshness check (skip entries from other active services)\n- Backup restore heartbeat (prevent BackupRestoreCancel)\n- reject_if_draining decorator on RPC entry points\n- signal_shutdown / cleanup_threadpool on ThreadPoolManager\n\nTest infrastructure:\n- BaseVolumeTestCase: shutdown executor in addCleanup to prevent\n  non-daemon ThreadPoolExecutor threads from blocking stestr exit\n- test_volume_cleanup: sync _add_to_threadpool mock + service_down_time\u003d0\n- test_admin_actions: explicit RPC server teardown in test tearDown\n- test_migration: sync _add_to_threadpool for sqlite thread safety\n- _common_cleanup: deregister fake transport RPC endpoints after stop()\n- _drain_pool: isinstance guard for Mock rpcserver objects\n\nChange-Id: Icdd28affc73fd34491b656a68410dce8e46264d4\nSigned-off-by: Walter Boring \u003cwaboring@hemna.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/d9afe47bfb56e3605a68f7c468fa82dacda71d93"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/cinder/commit/d9afe47bfb56e3605a68f7c468fa82dacda71d93"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"},{"label":"Review-Priority","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Review-Priority","description":"Review priority","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"-label:Review-Priority\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":[],"failing_atoms":["label:Review-Priority\u003dMIN"],"atom_explanations":{"label:Review-Priority\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}