)]}'
{".zuul.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[{"line_number":139,"context_line":"    post-run: playbooks/nova-live-migration/post-run.yaml"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"- job:"},{"line_number":142,"context_line":"    name: nova-graceful-shutdown"},{"line_number":143,"context_line":"    parent: devstack-multinode"},{"line_number":144,"context_line":"    description: |"},{"line_number":145,"context_line":"      Run Nova graceful shutdown tests."}],"source_content_type":"text/x-yaml","patch_set":14,"id":"12c1fd77_33918f7b","line":142,"updated":"2026-02-18 13:53:25.000000000","message":"We can avoid adding a new job if we want. The nova-live-migration job runs the playbooks/nova-live-migration/post-run.yaml post playbook to run destructive evacuation tests via https://github.com/openstack/nova/blob/master/roles/run-evacuate-hook/tasks/main.yaml . The test you add here is also a destructive one as it stops the compute service. So logically it fits to the same bucket.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":true,"context_lines":[{"line_number":139,"context_line":"    post-run: playbooks/nova-live-migration/post-run.yaml"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"- job:"},{"line_number":142,"context_line":"    name: nova-graceful-shutdown"},{"line_number":143,"context_line":"    parent: devstack-multinode"},{"line_number":144,"context_line":"    description: |"},{"line_number":145,"context_line":"      Run Nova graceful shutdown tests."}],"source_content_type":"text/x-yaml","patch_set":14,"id":"955ab8b6_e3245305","line":142,"in_reply_to":"12c1fd77_33918f7b","updated":"2026-02-19 02:58:09.000000000","message":"I saw that but did not want to expand the scope of that job because these tests are specifically for the graceful shutdown and not just limited to the live migration. I am expanding it for cold migration also in further changes in this series or may add more operations back and forth to test during shutdown.\n\nAlso, we run nova-live-migration in other repo like tempest but we do not need to run the graceful shutdown tests there as there is no tempest tests included in graceful shutdown testing.\n\nI am not sure if we need to run it in both check and gate pipeline so let me remove it from gate. Later if we think we do not need to run it on every change then we can move it to periodic pipeline but while we implement the graceful shutdown, it is good to run in check pipeline.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ec3659f48c223fa85e463423e7502ce34d1fdbc4","unresolved":false,"context_lines":[{"line_number":139,"context_line":"    post-run: playbooks/nova-live-migration/post-run.yaml"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"- job:"},{"line_number":142,"context_line":"    name: nova-graceful-shutdown"},{"line_number":143,"context_line":"    parent: devstack-multinode"},{"line_number":144,"context_line":"    description: |"},{"line_number":145,"context_line":"      Run Nova graceful shutdown tests."}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ea521f55_5d696eef","line":142,"in_reply_to":"955ab8b6_e3245305","updated":"2026-02-19 11:55:38.000000000","message":"OK good point that we are expanding this to other operations as well and each of them will be destructive. I missed that. I agree to keep it separate.\n\nOn the frequency I would run it on all the changes if CI resources permits.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"d370457d7f543c2ee4f55ca3603257d677ce953e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"c8b6984c_1af8512e","updated":"2026-02-16 20:17:07.000000000","message":"unit/functional test timeout is valid. I need to mock the manager graceful_shutodwn() method which takes 150 sec wait by default.","commit_id":"437ada1b0737a2748e38be8c645d79bcb038b27f"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"22e4758cbc755609ab165ab1c5e04cb2e1b5f666","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"3016fbb3_cd32369f","updated":"2026-02-17 02:50:32.000000000","message":"still testing it","commit_id":"45aed18f8dfac823a93d46e94fbe6c2b06a91dc4"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":14,"id":"4765d2f1_5dc55013","updated":"2026-02-18 13:53:25.000000000","message":"I have couple of comments inline\n\nJob log has some errors https://zuul.opendev.org/t/openstack/build/2ffb828e9d1e445db7482f81ac4216e6/log/job-output.txt#30191-30195","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":14,"id":"934e7a2e_74a10855","in_reply_to":"4765d2f1_5dc55013","updated":"2026-02-19 02:58:09.000000000","message":"yeah, error were from sourcing lib/tempest which we need only buide_timeout. We can define the timeout in script itself.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5023effb88c3ada305ed9112b8bb823a64c2c1f6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"0f5ed7bc_a640a9a1","updated":"2026-02-24 02:01:47.000000000","message":"Below are the logs snipshot to see the graceful shutdown worklfow:\n\n*Two RPC server created for compute service: @ Feb 23 21:57:29*\n\n- Feb 23 21:57:29.018274 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [None req-3c30d9c6-a812-46d1-ae6c-b31ca8a51b4f None None] Creating RPC server for service: nova-compute on topic: compute {{(pid\u003d39207) start /opt/stack/nova/nova/service.py:195}}\nFeb 23 21:57:29.041137 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [None req-3c30d9c6-a812-46d1-ae6c-b31ca8a51b4f None None] Creating 2nd RPC server for service: nova-compute on topic: compute-alt {{(pid\u003d39207) start /opt/stack/nova/nova/service.py:211}}\n\n- https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#9188\n\n*Shutdown is initiated, it stop first RPC server and wait for manager to finish the live migration: @ Feb 23 21:58:22*\n\n- Feb 23 21:58:22.345397 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service graceful shutdown started. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:324}}\nFeb 23 21:58:22.345397 np83a88d0cbc414 nova-compute[39207]:DEBUG nova.service [-] nova-compute service stopping RPC server on topic: compute {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:313}}\nFeb 23 21:58:23.386034 np83a88d0cbc414 nova-compute[39207]: DEBUG ovsdbapp.backend.ovs_idl.vlog [-] [POLLIN] on fd 22 {{(pid\u003d39207) __log_wakeup /opt/stack/data/venv/lib/python3.12/site-packages/ovs/poller.py:263}}\nFeb 23 21:58:23.723714 np83a88d0cbc414 nova-compute[39207]: WARNING amqp [-] Received method (60, 30) during closing channel 1. This method will be ignored\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopped RPC server on topic: compute {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:317}}\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute manager graceful shutdown started. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:332}}\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.compute.manager [-] Compute service manager is waiting for 160 seconds to finish in-progress tasks {{(pid\u003d39207) graceful_shutdown\n\n- https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#9539\n\n*Live migration is finished: @ Feb 23 21:58:44*\n\n- Feb 23 21:58:44.550708 np83a88d0cbc414 nova-compute[39207]: INFO nova.compute.manager [None req-59d1eb6f-4e50-4249-84f3-bf4dde91adaf demo admin] [instance: d0cd6cd2-8172-4c4f-882b-fe526d5c600b] Migrating instance to npb9fff9b95e354 finished successfully.\n\n- https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#10099\n\n*Manager finished the wait/live migration and 2nd RPC server is stopped and shutdown is done gracefully. Nova compute service is stopped. @ Feb 23 22:01:03*\n\n- Feb 23 22:01:03.728947 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute manager graceful shutdown finished. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:335}}\nFeb 23 22:01:03.729100 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopping RPC server on topic: compute-alt {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:313}}\nFeb 23 22:01:04.055277 np83a88d0cbc414 nova-compute[39207]: WARNING amqp [-] Received method (60, 30) during closing channel 1. This method will be ignored\nFeb 23 22:01:04.056816 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopped RPC server on topic: compute-alt {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:317}}\nFeb 23 22:01:04.056962 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service graceful shutdown finished. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:348}}\n\n- https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#10237","commit_id":"7ffc20516bec4aca67da90c3bd2b45aff628955e"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"47fa6c50cfa58ea4d49b9c2d0ed1013ed45cf9c3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"b2f828ed_0690aadf","updated":"2026-02-24 01:30:00.000000000","message":"recheck server fault in nova-lvm job","commit_id":"7ffc20516bec4aca67da90c3bd2b45aff628955e"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a98f346add2bbd4923fa6ca65d1f892c97c14de0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"7b365543_76f16ae5","updated":"2026-02-24 15:35:31.000000000","message":"Thanks","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"45835bb4231307469755a39e5eb155c558f1f062","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"d3748eae_efdbf273","updated":"2026-02-25 16:32:30.000000000","message":"droping back to +1 i really think if we are going to claim we suprpot this for move opreration you need funcitonl test to assert the correct behvior during the move operations.\nyou have not tested hwat happens if you stop the souce nova-compute after we call migrate_to_uri3\n\ni.e. started the live migration at teh hypervioer/libvirt level\n\nits entrily posible for that to compelte and the vm to mvoe to the destination and the db to get out of sync since we are making no attempt to abbort in progress live migrations in this change at the hypervisor level. that only planned for spec 3 in the backlog spec so we should not document or suggest it safe to use graceful restarte when tehre is a ongoing live migration at least not without declaring the feature experimetnal and having an explcit warning.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"e89bad75_36dc4d19","updated":"2026-02-25 14:51:14.000000000","message":"i do have some reservtation with how the new job works and other comments inline but i think thos ecan be adresse later.\nim going to review the rest fo the series and loop back once the eventlet patchs have merged.\n\nthe build history for the job is about 50% passing but the last few runs have all pasted so we will need to review it","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7709efbf5ca744df9e5680b05d1ff92f18cd948f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"7154e736_90fdc064","updated":"2026-02-25 16:33:57.000000000","message":"if we update the release note and documetn/call this feature experimental im ok to restore my +2 and adress the lack fo functional test and the remiaining edge cases next cycle","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f3a1fa5ff270cb311988a46c9d62a3cf8aaff6b0","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":26,"id":"4ae7dd2e_c47fd666","in_reply_to":"4a217b67_e82b9258","updated":"2026-02-25 20:11:05.000000000","message":"this is something we will have to adress before we consider this to not be experiment in my opion\ni do not think the integration test and unit tests are sufficient for this really.\nthe integration test are racy and skip if they lose the race.\n\nas i said we can address this next cycle but form my perspective the functional test were the most important test to implement out of the testing we discussed in the spec.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a9e4fa7aea9aa4918074ba35b040ee54cb56805d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":26,"id":"4a217b67_e82b9258","in_reply_to":"d3748eae_efdbf273","updated":"2026-02-25 19:46:39.000000000","message":"Before writting the integration tests, I checked if functional tests are enough or not but they are not. Functional test use fake rabiitmq driver and these changes are all about how RPC servers stop/wait and how compute talk to each other via 2nd RPC server so testing it witn fake message queue driver is not enough.\n\nIntegration tests are working fine, migrations in upstream CI takes ~23 sec and test is able to raise SIGTERM before that. we can see the workflow from the logs:\n\npasting it here too for quick ref:\n\n---------------------------\n\n1. Two RPC server created for compute service: @ Feb 23 21:57:29\n\n   - Feb 23 21:57:29.018274 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [None req-3c30d9c6-a812-46d1-ae6c-b31ca8a51b4f None None] Creating RPC server for service: nova-compute on topic: compute {{(pid\u003d39207) start /opt/stack/nova/nova/service.py:195}}\nFeb 23 21:57:29.041137 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [None req-3c30d9c6-a812-46d1-ae6c-b31ca8a51b4f None None] Creating 2nd RPC server for service: nova-compute on topic: compute-alt {{(pid\u003d39207) start /opt/stack/nova/nova/service.py:211}}\n\n      - https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#9188\n\n2. Shutdown is initiated, it stop first RPC server and wait for manager to finish the live migration: @ Feb 23 21:58:22\n\n   - Feb 23 21:58:22.345397 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service graceful shutdown started. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:324}}\nFeb 23 21:58:22.345397 np83a88d0cbc414 nova-compute[39207]:DEBUG nova.service [-] nova-compute service stopping RPC server on topic: compute {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:313}}\nFeb 23 21:58:23.386034 np83a88d0cbc414 nova-compute[39207]: DEBUG ovsdbapp.backend.ovs_idl.vlog [-] [POLLIN] on fd 22 {{(pid\u003d39207) __log_wakeup /opt/stack/data/venv/lib/python3.12/site-packages/ovs/poller.py:263}}\nFeb 23 21:58:23.723714 np83a88d0cbc414 nova-compute[39207]: WARNING amqp [-] Received method (60, 30) during closing channel 1. This method will be ignored\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopped RPC server on topic: compute {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:317}}\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute manager graceful shutdown started. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:332}}\nFeb 23 21:58:23.725746 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.compute.manager [-] Compute service manager is waiting for 160 seconds to finish in-progress tasks {{(pid\u003d39207) graceful_shutdown\n\n      - https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#9539\n\n3. Live migration is finished: @ Feb 23 21:58:44\n\n   - Feb 23 21:58:44.550708 np83a88d0cbc414 nova-compute[39207]: INFO nova.compute.manager [None req-59d1eb6f-4e50-4249-84f3-bf4dde91adaf demo admin] [instance: d0cd6cd2-8172-4c4f-882b-fe526d5c600b] Migrating instance to npb9fff9b95e354 finished successfully.\n\n      - https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#10099\n\n4. Manager finished the wait/live migration and 2nd RPC server is stopped and shutdown is done gracefully. Nova compute service is stopped. @ Feb 23 22:01:03\n\n   - Feb 23 22:01:03.728947 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute manager graceful shutdown finished. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:335}}\nFeb 23 22:01:03.729100 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopping RPC server on topic: compute-alt {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:313}}\nFeb 23 22:01:04.055277 np83a88d0cbc414 nova-compute[39207]: WARNING amqp [-] Received method (60, 30) during closing channel 1. This method will be ignored\nFeb 23 22:01:04.056816 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service stopped RPC server on topic: compute-alt {{(pid\u003d39207) _shutdown_rpc_server /opt/stack/nova/nova/service.py:317}}\nFeb 23 22:01:04.056962 np83a88d0cbc414 nova-compute[39207]: DEBUG nova.service [-] nova-compute service graceful shutdown finished. {{(pid\u003d39207) stop /opt/stack/nova/nova/service.py:348}}\n\n    - https://zuul.opendev.org/t/openstack/build/f1a02622045640bdb6554d410d69b0e8/log/compute1/logs/screen-n-cpu.txt#10237","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"91c8c866630adb7f2ec37e3255fdfcf3f7d5f62e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"d25d4ceb_14983246","updated":"2026-02-26 09:25:27.000000000","message":"still good","commit_id":"d5ffb58a8d55804fd46df10ce841362ed2851640"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"f3f1e8b4702e90ef7ee29f0552fc1bfb41a8cb43","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"03b3dfbd_9ac06e07","updated":"2026-02-26 10:55:31.000000000","message":"we can give it a go","commit_id":"d5ffb58a8d55804fd46df10ce841362ed2851640"}],"nova/compute/rpcapi.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"8cb78ae45bfecd4351b32c4e12e71fa2e73ba584","unresolved":true,"context_lines":[{"line_number":1550,"context_line":""},{"line_number":1551,"context_line":"    def external_instance_event(self, ctxt, instances, events, host\u003dNone):"},{"line_number":1552,"context_line":"        instance \u003d instances[0]"},{"line_number":1553,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1554,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1555,"context_line":"        client \u003d self.router.client(ctxt)"},{"line_number":1556,"context_line":"        cctxt \u003d self.prepare_for_rpc_ops("},{"line_number":1557,"context_line":"            client,"}],"source_content_type":"text/x-python","patch_set":1,"id":"c27f79a3_5a0f5bff","line":1554,"range":{"start_line":1553,"start_character":0,"end_line":1554,"end_character":40},"updated":"2026-02-05 13:46:10.000000000","message":"duplicate lines","commit_id":"d1a75b0714a7bcaa60666a3c79e96d8df0c3ab99"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"cbd02c5c5bfff6e04548361e47f10b4a34605b82","unresolved":false,"context_lines":[{"line_number":1550,"context_line":""},{"line_number":1551,"context_line":"    def external_instance_event(self, ctxt, instances, events, host\u003dNone):"},{"line_number":1552,"context_line":"        instance \u003d instances[0]"},{"line_number":1553,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1554,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1555,"context_line":"        client \u003d self.router.client(ctxt)"},{"line_number":1556,"context_line":"        cctxt \u003d self.prepare_for_rpc_ops("},{"line_number":1557,"context_line":"            client,"}],"source_content_type":"text/x-python","patch_set":1,"id":"b17866da_7eb86811","line":1554,"range":{"start_line":1553,"start_character":0,"end_line":1554,"end_character":40},"in_reply_to":"1added3a_9df4507a","updated":"2026-02-17 22:53:45.000000000","message":"Done","commit_id":"d1a75b0714a7bcaa60666a3c79e96d8df0c3ab99"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"67912196b6b9ec98e17822c99139e8ad27dae4d7","unresolved":true,"context_lines":[{"line_number":1550,"context_line":""},{"line_number":1551,"context_line":"    def external_instance_event(self, ctxt, instances, events, host\u003dNone):"},{"line_number":1552,"context_line":"        instance \u003d instances[0]"},{"line_number":1553,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1554,"context_line":"        version \u003d self._ver(ctxt, \u00275.0\u0027)"},{"line_number":1555,"context_line":"        client \u003d self.router.client(ctxt)"},{"line_number":1556,"context_line":"        cctxt \u003d self.prepare_for_rpc_ops("},{"line_number":1557,"context_line":"            client,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1added3a_9df4507a","line":1554,"range":{"start_line":1553,"start_character":0,"end_line":1554,"end_character":40},"in_reply_to":"c27f79a3_5a0f5bff","updated":"2026-02-17 01:11:53.000000000","message":"Done","commit_id":"d1a75b0714a7bcaa60666a3c79e96d8df0c3ab99"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[{"line_number":645,"context_line":"        # topic. If a shutdown is initiated on the destination compute, the"},{"line_number":646,"context_line":"        # RPC server for the \u0027compute\u0027 topic will be stopped. If a live"},{"line_number":647,"context_line":"        # migration request arrives after that, the destination compute node"},{"line_number":648,"context_line":"        # should not take it."},{"line_number":649,"context_line":"        cctxt \u003d client.prepare(server\u003ddestination, version\u003dversion,"},{"line_number":650,"context_line":"                               call_monitor_timeout\u003dCONF.rpc_response_timeout,"},{"line_number":651,"context_line":"                               timeout\u003dCONF.long_rpc_timeout)"}],"source_content_type":"text/x-python","patch_set":26,"id":"12a9dea4_7b5699ff","line":648,"updated":"2026-02-25 14:51:14.000000000","message":"ok i guess that does make sense as a way to detech that.\nthis si a good callout.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[{"line_number":952,"context_line":"        # stage, both the source and destination compute have already confirmed"},{"line_number":953,"context_line":"        # that live migration can proceed. If the shutdown is initiated after"},{"line_number":954,"context_line":"        # that, the compute should finish the live migration using the"},{"line_number":955,"context_line":"        # \u0027compute-alt\u0027 RPC server."},{"line_number":956,"context_line":"        cctxt \u003d self.prepare_for_alt_rpcserver("},{"line_number":957,"context_line":"            client, server\u003dhost, version\u003dversion)"},{"line_number":958,"context_line":"        cctxt.cast(ctxt, \u0027live_migration\u0027, instance\u003dinstance,"}],"source_content_type":"text/x-python","patch_set":26,"id":"ee0b3b69_f5842888","line":955,"updated":"2026-02-25 14:51:14.000000000","message":"if the destination is asked to shutdown it could check that at the start of processing this rpc and imideatly abort the live migration.\n\nthat is something we can change in the future so its not a issue for now this is still a good improvement.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"}],"nova/objects/service.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"8cb78ae45bfecd4351b32c4e12e71fa2e73ba584","unresolved":false,"context_lines":[{"line_number":251,"context_line":"    {\u0027compute_rpc\u0027: \u00276.4\u0027},"},{"line_number":252,"context_line":"    # Version 71: Compute RPC v6.5:"},{"line_number":253,"context_line":"    # Add 2nd RPC server for compute service"},{"line_number":254,"context_line":"    {\u0027compute_rpc\u0027: \u00276.5\u0027},"},{"line_number":255,"context_line":")"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"# This is the version after which we can rely on having a persistent"}],"source_content_type":"text/x-python","patch_set":1,"id":"f23b3003_8759164a","line":254,"updated":"2026-02-05 13:46:10.000000000","message":"++","commit_id":"d1a75b0714a7bcaa60666a3c79e96d8df0c3ab99"}],"nova/test.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"8cb78ae45bfecd4351b32c4e12e71fa2e73ba584","unresolved":false,"context_lines":[{"line_number":508,"context_line":"        def patch_stop(*a, **k):"},{"line_number":509,"context_line":"            self._service_fixture_count[name] -\u003d 1"},{"line_number":510,"context_line":"            with mock.patch.object("},{"line_number":511,"context_line":"                svc.service.manager, \u0027graceful_shutdown\u0027, return_value\u003dTrue):"},{"line_number":512,"context_line":"                return real_stop(*a, **k)"},{"line_number":513,"context_line":"        self.useFixture(fixtures.MockPatchObject("},{"line_number":514,"context_line":"            svc.service, \u0027stop\u0027, patch_stop))"}],"source_content_type":"text/x-python","patch_set":1,"id":"cc439d7e_6c4b5fc6","line":511,"updated":"2026-02-05 13:46:10.000000000","message":"yeah good point as a temporary measure. When we will have logic (and not just sleep) in that hook then we might want to keep running the hook.","commit_id":"d1a75b0714a7bcaa60666a3c79e96d8df0c3ab99"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[{"line_number":464,"context_line":"        # \u0027topic\u0027 from service binary name. For compute service, we need"},{"line_number":465,"context_line":"        # to create the 2nd RPC server which will be done by pass the"},{"line_number":466,"context_line":"        # \u0027topic_alt\u0027 explicitly."},{"line_number":467,"context_line":"        if name \u003d\u003d \u0027compute\u0027 and \u0027topic_alt\u0027 not in kwargs:"},{"line_number":468,"context_line":"            kwargs[\u0027topic_alt\u0027] \u003d compute_rpcapi.RPC_TOPIC_ALT"},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"        if name \u003d\u003d \u0027compute\u0027 and self.USES_DB:"},{"line_number":471,"context_line":"            # NOTE(danms): We need to create the HostMapping first, because"}],"source_content_type":"text/x-python","patch_set":26,"id":"a8fd8fc5_1ef160d6","line":468,"range":{"start_line":467,"start_character":0,"end_line":468,"end_character":62},"updated":"2026-02-25 14:51:14.000000000","message":"nit: i woudl have proply used set default here\n\n\n```suggestion\n        if name \u003d\u003d \u0027compute\u0027:\n            kwargs.set_default(\u0027topic_alt\u0027,compute_rpcapi.RPC_TOPIC_ALT)\n```\n\nbut this is fine.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"}],"releasenotes/notes/nova-services-graceful-shutdown-564a321e2769152d.yaml":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"features:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Nova services now support graceful shutdown on ``SIGTERM``. When a service"},{"line_number":5,"context_line":"    receives ``SIGTERM``, it will stop accepting new RPC requests and wait for"},{"line_number":6,"context_line":"    in-progress tasks to reach a safe termination point."},{"line_number":7,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":26,"id":"9dcc24cb_7d244f48","line":4,"range":{"start_line":4,"start_character":3,"end_line":4,"end_character":48},"updated":"2026-02-25 14:51:14.000000000","message":"i dont think this is a enought to requried a respine but this is proably a little stronger then i would say\n\ni think until we complete all the phases in the backlog spec this is not entirly ture\n\nit is more graceful then before for sure but i would not conditioner the current shutdown support to be graceful yet.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a9e4fa7aea9aa4918074ba35b040ee54cb56805d","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"features:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Nova services now support graceful shutdown on ``SIGTERM``. When a service"},{"line_number":5,"context_line":"    receives ``SIGTERM``, it will stop accepting new RPC requests and wait for"},{"line_number":6,"context_line":"    in-progress tasks to reach a safe termination point."},{"line_number":7,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":26,"id":"d75f8349_07d1b781","line":4,"range":{"start_line":4,"start_character":3,"end_line":4,"end_character":48},"in_reply_to":"9dcc24cb_7d244f48","updated":"2026-02-25 19:46:39.000000000","message":"the next phases are surly a improvement for better task tracking but still this phase solve the common problem user has faced \"build instance stop if shutdown is initiated\" + all migration cases etc.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"    * Live migration"},{"line_number":15,"context_line":"    * Server external Event"},{"line_number":16,"context_line":"    * Get Console output"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"    Nova added two new configuration options which will control this behavior:"},{"line_number":19,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":26,"id":"4439af7e_f9895d14","line":16,"updated":"2026-02-25 14:51:14.000000000","message":"so you have caveated this by calling out the limited set of operation\nwhich is why im ok with proceeding with this release note as is but i woudl prefer if we maded it clear that this si still a work in progress and an incremental impovemnt rahter\n\n\ni woudl over all clasify the graceful shutdown feature as experimental similar to runnign nova in threaded mode and we will need to continue improving it over the next cycle or two as i dont think this is enough to start relaying on in production.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a9e4fa7aea9aa4918074ba35b040ee54cb56805d","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"    * Live migration"},{"line_number":15,"context_line":"    * Server external Event"},{"line_number":16,"context_line":"    * Get Console output"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"    Nova added two new configuration options which will control this behavior:"},{"line_number":19,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":26,"id":"32cde42c_5cf5cbe0","line":16,"in_reply_to":"4439af7e_f9895d14","updated":"2026-02-25 19:46:39.000000000","message":"sure, I am not against of calling that out in releasenotes and once we have all phase completed then we can make it more exact on what all operations it will complete and what all still risky.","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9fe67d7bb83b6cc7b4c0bda07f80a053cd36e576","unresolved":true,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":"    We plan to improve the graceful shutdown in future releases by task"},{"line_number":32,"context_line":"    tracking and transitioning resources to a recoverable state. Until then,"},{"line_number":33,"context_line":"    this feature is experimental."},{"line_number":34,"context_line":"upgrade:"},{"line_number":35,"context_line":"  - |"},{"line_number":36,"context_line":"    The default value of ``[DEFAULT]/graceful_shutdown_timeout`` has been"}],"source_content_type":"text/x-yaml","patch_set":28,"id":"4b77978e_0ed8e047","line":33,"updated":"2026-02-25 20:53:26.000000000","message":"+1","commit_id":"d5ffb58a8d55804fd46df10ce841362ed2851640"}],"roles/restart-and-verify-compute-service/files/verify_compute_service.sh":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[{"line_number":12,"context_line":"  sleep 1"},{"line_number":13,"context_line":"  count\u003d$((count+1))"},{"line_number":14,"context_line":"  if [ ${count} -eq 30 ]; then"},{"line_number":15,"context_line":"    echo \"Timed out waiting for subnode compute service to come back up\""},{"line_number":16,"context_line":"    exit 5"},{"line_number":17,"context_line":"  fi"},{"line_number":18,"context_line":"  status\u003d$(openstack compute service list --host \"${COMPUTE_HOST}\" --service nova-compute -f value -c State)"}],"source_content_type":"text/x-sh","patch_set":14,"id":"aa8ebe11_def2421b","line":15,"updated":"2026-02-18 13:53:25.000000000","message":"even though we know that this script will only be called with the subnode name the script itself is generic so I suggest to just use the $COMPUTE_HOST in the log statement.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[{"line_number":12,"context_line":"  sleep 1"},{"line_number":13,"context_line":"  count\u003d$((count+1))"},{"line_number":14,"context_line":"  if [ ${count} -eq 30 ]; then"},{"line_number":15,"context_line":"    echo \"Timed out waiting for subnode compute service to come back up\""},{"line_number":16,"context_line":"    exit 5"},{"line_number":17,"context_line":"  fi"},{"line_number":18,"context_line":"  status\u003d$(openstack compute service list --host \"${COMPUTE_HOST}\" --service nova-compute -f value -c State)"}],"source_content_type":"text/x-sh","patch_set":14,"id":"181ffd5b_9979aee7","line":15,"in_reply_to":"aa8ebe11_def2421b","updated":"2026-02-19 02:58:09.000000000","message":"Done","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"}],"roles/restart-and-verify-compute-service/tasks/main.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[{"line_number":7,"context_line":"  become: true"},{"line_number":8,"context_line":"  systemd:"},{"line_number":9,"context_line":"    name: devstack@n-cpu"},{"line_number":10,"context_line":"    state: restarted"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"- name: Wait for subnode compute service to be ready"},{"line_number":13,"context_line":"  become: true"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ff1aceed_42885011","line":10,"updated":"2026-02-18 13:53:25.000000000","message":"hm, this will trigger a stop and then a start if the service is still running. I think we want to assume here that the service is successfully stopped and therefore only trigger a start by asking for `state: started`\n\nOr even better verify after the sleep that the service is in stopped state as if not then graceful shutdown failed somehow\n\n// later\n\nI would move the sleep and the stopped state verification to the live migration role as that role sends the SIGTERM so that should verify the effect of it too. And I would use \"started\" here to start up the service","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[{"line_number":7,"context_line":"  become: true"},{"line_number":8,"context_line":"  systemd:"},{"line_number":9,"context_line":"    name: devstack@n-cpu"},{"line_number":10,"context_line":"    state: restarted"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"- name: Wait for subnode compute service to be ready"},{"line_number":13,"context_line":"  become: true"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"438bb47b_21e9471a","line":10,"in_reply_to":"ff1aceed_42885011","updated":"2026-02-19 02:58:09.000000000","message":"yeah, I think that is good point to verify if service is stoped and if not then fail. done","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"}],"roles/run-graceful-shutdown-live-migration/files/verify_graceful_shutdown_live_migration.sh":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":false,"context_lines":[{"line_number":36,"context_line":"echo \"Server graceful-shutdown-server is ACTIVE on ${host}\""},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"# Cleanup test server"},{"line_number":39,"context_line":"openstack server delete --wait graceful-shutdown-server"}],"source_content_type":"text/x-sh","patch_set":14,"id":"a701b646_953fddf2","line":39,"updated":"2026-02-18 13:53:25.000000000","message":"OK this should work as the VM is migrated out from the shutting down node and now on the controller that is not in shutdown state.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[{"line_number":36,"context_line":"echo \"Server graceful-shutdown-server is ACTIVE on ${host}\""},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"# Cleanup test server"},{"line_number":39,"context_line":"openstack server delete --wait graceful-shutdown-server"}],"source_content_type":"text/x-sh","patch_set":14,"id":"ae1da7c2_4ba731a5","line":39,"in_reply_to":"a701b646_953fddf2","updated":"2026-02-19 02:58:09.000000000","message":"yeah, server is on controller now and delete should happen successfully.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"}],"roles/run-graceful-shutdown-live-migration/tasks/main.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[{"line_number":1,"context_line":"- name: Start live migration of test server"},{"line_number":2,"context_line":"  become: true"},{"line_number":3,"context_line":"  become_user: stack"},{"line_number":4,"context_line":"  shell: \"/opt/stack/nova/roles/run-graceful-shutdown-live-migration/files/start_live_migration.sh\""},{"line_number":5,"context_line":"  environment:"},{"line_number":6,"context_line":"    SUBNODE_HOSTNAME: \"{{ hostvars[\u0027compute1\u0027][\u0027ansible_hostname\u0027] }}\""},{"line_number":7,"context_line":"    CONTROLLER_HOSTNAME: \"{{ hostvars[\u0027controller\u0027][\u0027ansible_hostname\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":14,"id":"0597f8b9_1544a0a2","line":4,"updated":"2026-02-18 13:53:25.000000000","message":"I feel like ansible should be able to use the files from the role without hardcoding the full path to it.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[{"line_number":1,"context_line":"- name: Start live migration of test server"},{"line_number":2,"context_line":"  become: true"},{"line_number":3,"context_line":"  become_user: stack"},{"line_number":4,"context_line":"  shell: \"/opt/stack/nova/roles/run-graceful-shutdown-live-migration/files/start_live_migration.sh\""},{"line_number":5,"context_line":"  environment:"},{"line_number":6,"context_line":"    SUBNODE_HOSTNAME: \"{{ hostvars[\u0027compute1\u0027][\u0027ansible_hostname\u0027] }}\""},{"line_number":7,"context_line":"    CONTROLLER_HOSTNAME: \"{{ hostvars[\u0027controller\u0027][\u0027ansible_hostname\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":14,"id":"2d3ef090_3b7b06ec","line":4,"in_reply_to":"0597f8b9_1544a0a2","updated":"2026-02-19 02:58:09.000000000","message":"Done","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20cc77126c49688cd10b8b6a41741355a8b5dbba","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":14,"id":"c69e7c9d_6a757a29","line":20,"updated":"2026-02-18 13:53:25.000000000","message":"1. I suggest to make a verification step here about the effect of the SIGTERM. So here we know that the live migration succeeded even though we sent a SIGTERM, but we can also check that it is not succeeded just because the signal was  ignored by the subnode. I suggest to verify here that the compute service on the subnode is shut down successfully as a result of our SIGTERM\n\n2. As a followup it would be nice to try to stat a second live migration *after* the subnode is in graceful shutting down state and verify that such live migration remains queued until the subnode is started up again.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a25c553c39147148f49843fb95535e7d67f6d679","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":14,"id":"c10617f4_b8b39efb","line":20,"in_reply_to":"c69e7c9d_6a757a29","updated":"2026-02-19 02:58:09.000000000","message":"1) done.\n2) yeah, we can add such scenario here and for more operations not just live migration. that is why I defined a separate job for the gradceful shutdown.","commit_id":"56c1cadbaadcf409d725a6c2c3f0110bc6bcb22c"}],"roles/run-graceful-shutdown-tests/files/start_and_verify_compute_service.sh":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"5274bccbfa98294f18e6c4e71ae3e47d7096e0a5","unresolved":true,"context_lines":[{"line_number":12,"context_line":"  local count\u003d0"},{"line_number":13,"context_line":"  local status"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"  status\u003d$(ssh \"${host}\" systemctl is-active devstack@n-cpu || true)"},{"line_number":16,"context_line":"  while [ \"${status}\" !\u003d \"${expected}\" ]; do"},{"line_number":17,"context_line":"    sleep 5"},{"line_number":18,"context_line":"    count\u003d$((count+1))"}],"source_content_type":"text/x-sh","patch_set":22,"id":"123b367d_653c7387","line":15,"updated":"2026-02-23 17:31:37.000000000","message":"nit: this is repeated now 3 times in this file. It might worth to add a function for it","commit_id":"5986f7af7abe6018e13b78504afcc67f98047aa7"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"12d2a4c1cb265225b86df8906649150948f7d72f","unresolved":false,"context_lines":[{"line_number":12,"context_line":"  local count\u003d0"},{"line_number":13,"context_line":"  local status"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"  status\u003d$(ssh \"${host}\" systemctl is-active devstack@n-cpu || true)"},{"line_number":16,"context_line":"  while [ \"${status}\" !\u003d \"${expected}\" ]; do"},{"line_number":17,"context_line":"    sleep 5"},{"line_number":18,"context_line":"    count\u003d$((count+1))"}],"source_content_type":"text/x-sh","patch_set":22,"id":"c302ef22_55880d3d","line":15,"in_reply_to":"123b367d_653c7387","updated":"2026-02-23 18:07:55.000000000","message":"Done","commit_id":"5986f7af7abe6018e13b78504afcc67f98047aa7"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"140607993be73a22a60749c387a61adab6c7d53a","unresolved":true,"context_lines":[{"line_number":8,"context_line":"get_service_status() {"},{"line_number":9,"context_line":"  local host\u003d$1"},{"line_number":10,"context_line":"  local status"},{"line_number":11,"context_line":"  status\u003d$(ssh \"${host}\" systemctl is-active devstack@n-cpu || true)"},{"line_number":12,"context_line":"  echo \"${status}\""},{"line_number":13,"context_line":"}"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-sh","patch_set":26,"id":"61f26565_651b0700","line":11,"range":{"start_line":11,"start_character":63,"end_line":11,"end_character":67},"updated":"2026-02-25 14:51:14.000000000","message":"im not sure we shoudl be doing that.\n\n\nif nova compute is not even deployed on the compute it woudl retru true.\n\nfor the curretnly limited usage it might be ok but we shoudl probaly revisit this","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a9e4fa7aea9aa4918074ba35b040ee54cb56805d","unresolved":true,"context_lines":[{"line_number":8,"context_line":"get_service_status() {"},{"line_number":9,"context_line":"  local host\u003d$1"},{"line_number":10,"context_line":"  local status"},{"line_number":11,"context_line":"  status\u003d$(ssh \"${host}\" systemctl is-active devstack@n-cpu || true)"},{"line_number":12,"context_line":"  echo \"${status}\""},{"line_number":13,"context_line":"}"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-sh","patch_set":26,"id":"9c12148e_c6e80275","line":11,"range":{"start_line":11,"start_character":63,"end_line":11,"end_character":67},"in_reply_to":"61f26565_651b0700","updated":"2026-02-25 19:46:39.000000000","message":"ack, I can remove that default in followup","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"}],"roles/run-graceful-shutdown-tests/tasks/main.yaml":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"540b7ec0064babcf767fd7b8c3128ea0ff2c3719","unresolved":true,"context_lines":[{"line_number":23,"context_line":"  pause:"},{"line_number":24,"context_line":"    seconds: 240"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"- name: Verify compute service is stopped after graceful shutdown"},{"line_number":27,"context_line":"  become: true"},{"line_number":28,"context_line":"  become_user: stack"},{"line_number":29,"context_line":"  script: \"{{ role_path }}/../start-and-verify-compute-service/files/verify_compute_service.sh {{ hostvars[\u0027compute1\u0027][\u0027ansible_hostname\u0027] }} down\""}],"source_content_type":"text/x-yaml","patch_set":18,"id":"f5606838_91ebc34c","line":29,"range":{"start_line":26,"start_character":0,"end_line":29,"end_character":147},"updated":"2026-02-19 07:45:04.000000000","message":"I can see graceful shutdown is completed before timeout and compute service is deactivated in compute logs but \u0027openstack compute service lsit\u0027 does give the updated status and report \u0027up\u0027.\n\nFeb 19 07:28:41.235744 npfe7fc9366bc94 nova-compute[39061]: DEBUG nova.service [-] nova-compute manager graceful shutdown finished. {{(pid\u003d39061) stop /opt/stack/nova/nova/service.py:335}}\nFeb 19 07:28:41.235824 npfe7fc9366bc94 nova-compute[39061]: DEBUG nova.service [-] nova-compute service stopping RPC server on topic: compute-alt {{(pid\u003d39061) _shutdown_rpc_server /opt/stack/nova/nova/service.py:313}}\nFeb 19 07:28:43.461515 npfe7fc9366bc94 nova-compute[39061]: DEBUG ovsdbapp.backend.ovs_idl.vlog [-] [POLLIN] on fd 22 {{(pid\u003d39061) __log_wakeup /opt/stack/data/venv/lib/python3.12/site-packages/ovs/poller.py:263}}\nFeb 19 07:28:48.462882 npfe7fc9366bc94 nova-compute[39061]: DEBUG ovsdbapp.backend.ovs_idl.vlog [-] [POLLIN] on fd 22 {{(pid\u003d39061) __log_wakeup /opt/stack/data/venv/lib/python3.12/site-packages/ovs/poller.py:263}}\nFeb 19 07:28:50.679152 npfe7fc9366bc94 nova-compute[39061]: WARNING amqp [-] Received method (60, 30) during closing channel 1. This method will be ignored\nFeb 19 07:28:50.679817 npfe7fc9366bc94 nova-compute[39061]: DEBUG nova.service [-] nova-compute service stopped RPC server on topic: compute-alt {{(pid\u003d39061) _shutdown_rpc_server /opt/stack/nova/nova/service.py:317}}\nFeb 19 07:28:50.679905 npfe7fc9366bc94 nova-compute[39061]: DEBUG nova.service [-] nova-compute service graceful shutdown finished. {{(pid\u003d39061) stop /opt/stack/nova/nova/service.py:348}}\nFeb 19 07:28:50.680784 npfe7fc9366bc94 nova-compute[39061]: DEBUG oslo_concurrency.lockutils [None req-45079629-c23b-4ca8-aa15-d9398cda0ebe None None] Acquiring lock \"singleton_lock\" {{(pid\u003d39061) lock /opt/stack/data/venv/lib/python3.12/site-packages/oslo_concurrency/lockutils.py:390}}\nFeb 19 07:28:50.680937 npfe7fc9366bc94 nova-compute[39061]: DEBUG oslo_concurrency.lockutils [None req-45079629-c23b-4ca8-aa15-d9398cda0ebe None None] Acquired lock \"singleton_lock\" {{(pid\u003d39061) lock /opt/stack/data/venv/lib/python3.12/site-packages/oslo_concurrency/lockutils.py:393}}\nFeb 19 07:28:50.681094 npfe7fc9366bc94 nova-compute[39061]: DEBUG oslo_concurrency.lockutils [None req-45079629-c23b-4ca8-aa15-d9398cda0ebe None None] Releasing lock \"singleton_lock\" {{(pid\u003d39061) lock /opt/stack/data/venv/lib/python3.12/site-packages/oslo_concurrency/lockutils.py:413}}\nFeb 19 07:28:51.272353 npfe7fc9366bc94 systemd[1]: devstack@n-cpu.service: Deactivated successfully.\nFeb 19 07:28:51.272453 npfe7fc9366bc94 systemd[1]: devstack@n-cpu.service: Unit process 41340 (privsep-helper) remains running after unit stopped.\nFeb 19 07:28:51.272516 npfe7fc9366bc94 systemd[1]: devstack@n-cpu.service: Unit process 41372 (privsep-helper) remains running after unit stopped.\nFeb 19 07:28:51.272786 npfe7fc9366bc94 systemd[1]: devstack@n-cpu.service: Consumed 12.175s CPU time, 434.4M memory peak, 0B memory swap peak.","commit_id":"3fede1fe0633b8113baefadb96884be193c10e45"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"fd50675240c3b796493860c53def52481ea9d469","unresolved":false,"context_lines":[{"line_number":23,"context_line":"  pause:"},{"line_number":24,"context_line":"    seconds: 240"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"- name: Verify compute service is stopped after graceful shutdown"},{"line_number":27,"context_line":"  become: true"},{"line_number":28,"context_line":"  become_user: stack"},{"line_number":29,"context_line":"  script: \"{{ role_path }}/../start-and-verify-compute-service/files/verify_compute_service.sh {{ hostvars[\u0027compute1\u0027][\u0027ansible_hostname\u0027] }} down\""}],"source_content_type":"text/x-yaml","patch_set":18,"id":"7de8a2f9_c3736abf","line":29,"range":{"start_line":26,"start_character":0,"end_line":29,"end_character":147},"in_reply_to":"c7ae8e62_d09452d4","updated":"2026-02-19 15:34:08.000000000","message":"ah devstack, I did not check and thought why it will override service_down_time  and only took care of default (60 sec). thanks for the pointer. will fix it.","commit_id":"3fede1fe0633b8113baefadb96884be193c10e45"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ec3659f48c223fa85e463423e7502ce34d1fdbc4","unresolved":true,"context_lines":[{"line_number":23,"context_line":"  pause:"},{"line_number":24,"context_line":"    seconds: 240"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"- name: Verify compute service is stopped after graceful shutdown"},{"line_number":27,"context_line":"  become: true"},{"line_number":28,"context_line":"  become_user: stack"},{"line_number":29,"context_line":"  script: \"{{ role_path }}/../start-and-verify-compute-service/files/verify_compute_service.sh {{ hostvars[\u0027compute1\u0027][\u0027ansible_hostname\u0027] }} down\""}],"source_content_type":"text/x-yaml","patch_set":18,"id":"c7ae8e62_d09452d4","line":29,"range":{"start_line":26,"start_character":0,"end_line":29,"end_character":147},"in_reply_to":"f5606838_91ebc34c","updated":"2026-02-19 11:55:38.000000000","message":"the up status in the API depends on the timeout and hearthbeat frequency configuration. \nI think devstack overrides the default making the detection of compute down slow.\n\nOur default\n```\n[DEFAULT]\nservice_down_time \u003d 60\nreport_interval \u003d 10\n```\ncompared to devstack https://zuul.opendev.org/t/openstack/build/8dce54c2b0c8437795a72294b8bc26b6/log/controller/logs/etc/nova/nova_conf.txt#21-22\n\nSo either we check the downness via systemd or we change the nova conf in this job to use the default service hearthbeat and timeout config.","commit_id":"3fede1fe0633b8113baefadb96884be193c10e45"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"5274bccbfa98294f18e6c4e71ae3e47d7096e0a5","unresolved":true,"context_lines":[{"line_number":14,"context_line":"        live_migrations_completed_before_sigterm: \"{{ start_live_migrations_result.rc \u003d\u003d 2 }}\""},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"    - name: Send SIGTERM to source compute to start the source compute graceful shutdown"},{"line_number":17,"context_line":"      when: not live_migrations_completed_before_sigterm"},{"line_number":18,"context_line":"      delegate_to: compute1"},{"line_number":19,"context_line":"      become: true"},{"line_number":20,"context_line":"      shell: \"kill -15 $(systemctl show devstack@n-cpu -p MainPID --value)\""}],"source_content_type":"text/x-yaml","patch_set":22,"id":"569b2e7a_9314e18d","line":17,"updated":"2026-02-23 17:31:37.000000000","message":"This means if the live migration is quick enough then we will skip the whole testing of the live migration. Obviously it is hard to test the whole scenario as it is depends on timing. So the question is which one we want \n* a stable test case that sometimes does not test the operation and passes silently\n* an unstable test case that sometimes fails when LM is too fast.\n\nPersonally prefer the latter as it is explicit. If that turns out two unstable then I think we need to give up the idea that we can test this on the gate in a voting job.","commit_id":"5986f7af7abe6018e13b78504afcc67f98047aa7"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"12d2a4c1cb265225b86df8906649150948f7d72f","unresolved":true,"context_lines":[{"line_number":14,"context_line":"        live_migrations_completed_before_sigterm: \"{{ start_live_migrations_result.rc \u003d\u003d 2 }}\""},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"    - name: Send SIGTERM to source compute to start the source compute graceful shutdown"},{"line_number":17,"context_line":"      when: not live_migrations_completed_before_sigterm"},{"line_number":18,"context_line":"      delegate_to: compute1"},{"line_number":19,"context_line":"      become: true"},{"line_number":20,"context_line":"      shell: \"kill -15 $(systemctl show devstack@n-cpu -p MainPID --value)\""}],"source_content_type":"text/x-yaml","patch_set":22,"id":"b5295cf2_1d5f6570","line":17,"in_reply_to":"569b2e7a_9314e18d","updated":"2026-02-23 18:07:55.000000000","message":"i cannot disagree with you on silently passing which is more wrong than not testing itself.\n\nIn all the run till now, I have not seen LM is completed before SIGTERM so let me do the latter one and also create little heavy VM with large flavor. And yes, if it fails then we can see how we can test it in gate.","commit_id":"5986f7af7abe6018e13b78504afcc67f98047aa7"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"2616c5a26c1b549e03c6f852c5dedf4305addbe5","unresolved":false,"context_lines":[{"line_number":14,"context_line":"        live_migrations_completed_before_sigterm: \"{{ start_live_migrations_result.rc \u003d\u003d 2 }}\""},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"    - name: Send SIGTERM to source compute to start the source compute graceful shutdown"},{"line_number":17,"context_line":"      when: not live_migrations_completed_before_sigterm"},{"line_number":18,"context_line":"      delegate_to: compute1"},{"line_number":19,"context_line":"      become: true"},{"line_number":20,"context_line":"      shell: \"kill -15 $(systemctl show devstack@n-cpu -p MainPID --value)\""}],"source_content_type":"text/x-yaml","patch_set":22,"id":"60d5a752_bfabed9a","line":17,"in_reply_to":"b5295cf2_1d5f6570","updated":"2026-02-23 20:34:43.000000000","message":"Done","commit_id":"5986f7af7abe6018e13b78504afcc67f98047aa7"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a98f346add2bbd4923fa6ca65d1f892c97c14de0","unresolved":false,"context_lines":[{"line_number":53,"context_line":"- name: Fail if any test is skipped"},{"line_number":54,"context_line":"  fail:"},{"line_number":55,"context_line":"    msg: \"One or more test is skipped due to operation is either completed or timed out before SIGTERM signal.\""},{"line_number":56,"context_line":"  when: live_migrations_completed_or_timeout"}],"source_content_type":"text/x-yaml","patch_set":26,"id":"04b039f9_45339263","line":56,"updated":"2026-02-24 15:35:31.000000000","message":"Thanks","commit_id":"18aad40f9db1afd377e2f6a54f95b9d54ab85443"}]}