)]}'
{"id":"openstack%2Fnova~851832","triplet_id":"openstack%2Fnova~master~I65534b52abfb1c6d331c73cc40413becdf3b05b1","project":"openstack/nova","branch":"master","topic":"bug/1982497","attention_set":{"9708":{"account":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"last_update":"2022-08-03 16:11:58.000000000","reason":"\u003cGERRIT_ACCOUNT_4393\u003e replied on the change","reason_account":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"}},"15334":{"account":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"last_update":"2022-08-02 09:37:00.000000000","reason":"\u003cGERRIT_ACCOUNT_9708\u003e replied on the change","reason_account":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"}},"8864":{"account":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"last_update":"2022-08-03 08:43:18.000000000","reason":"\u003cGERRIT_ACCOUNT_9708\u003e replied on the change","reason_account":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"}}},"removed_from_attention_set":{"4393":{"account":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"last_update":"2022-08-03 16:11:58.000000000","reason":"\u003cGERRIT_ACCOUNT_4393\u003e replied on the change","reason_account":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"}}},"hashtags":[],"change_id":"I65534b52abfb1c6d331c73cc40413becdf3b05b1","subject":"Do not mutate migration context for rollback_live_migration_at_destination","status":"NEW","created":"2022-08-02 09:35:43.000000000","updated":"2022-08-03 23:54:01.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":18,"unresolved_comment_count":3,"has_review_started":true,"meta_rev_id":"9368b4b11b3ab8ea3b9adc2d7d26f581cde51766","_number":851832,"virtual_id_number":851832,"owner":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":1,"date":"2022-08-03 16:35:24.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Review-Priority":{"all":[{"value":0,"permitted_voting_range":{"min":0,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"permitted_voting_range":{"min":0,"max":1},"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"}],"values":{" 0":"Default Priority","+1":"Contributor Review Promise","+2":"Core Review Promise"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"CC":[{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"}],"REVIEWER":[{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2022-08-02 09:37:00.000000000","updated_by":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"reviewer":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"state":"REVIEWER"},{"updated":"2022-08-02 09:37:00.000000000","updated_by":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"reviewer":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"state":"REVIEWER"},{"updated":"2022-08-02 09:37:09.000000000","updated_by":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"reviewer":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"state":"REVIEWER"},{"updated":"2022-08-02 10:48:54.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2022-08-02 11:26:42.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2022-08-02 13:46:38.000000000","updated_by":{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},"reviewer":{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2022-08-02 14:12:42.000000000","updated_by":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"reviewer":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"state":"CC"},{"updated":"2022-08-02 14:22:41.000000000","updated_by":{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"},"reviewer":{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"},"state":"CC"},{"updated":"2022-08-02 17:36:52.000000000","updated_by":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"reviewer":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"state":"CC"}],"messages":[{"id":"6ba5bb243b8e5a733032d5ac2cbdbef442146bd6","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-02 09:35:43.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"84af6bf616f5d14a9ed79553f21340c42c37f1fa","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-02 09:36:19.000000000","message":"Patch Set 1: Workflow-1\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"b06a32737e1b5cd879c7e1f0198f0dce739ef418","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 10:48:54.000000000","message":"Patch Set 1:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/a6e8b2e66e404354bf0588e58bf23a87\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/8cbd0c44f95c416c88247950c609c0e9 : FAILURE in 47m 15s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/7b2a91b4f7f14759ad78ed8ba6ad21fb : FAILURE in 28m 58s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"0ab51c0a7f3f22f28ff224c8ac798d57f9e07080","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 11:26:42.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/81da86b326b8455fb906381ba1d0c0c4\n\n- tempest-integrated-compute https://zuul.opendev.org/t/openstack/build/ebd00e21f663433e834c91fa82f0e8c8 : SUCCESS in 1h 24m 10s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/bd6503f397d04580bc891b932317fc95 : SUCCESS in 54m 36s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/1adc48785a1e4fa3894a5861b5fd13bd : FAILURE in 15m 53s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/482b6e3faad144f391da0ae52ad3c6eb : SUCCESS in 11m 26s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/48a848346fc74be78f6b51cf421c36e7 : FAILURE in 13m 36s\n- openstack-tox-py39 https://zuul.opendev.org/t/openstack/build/309bad332ffe431397fbf5da3bf911bb : FAILURE in 12m 39s\n- openstack-tox-py310 https://zuul.opendev.org/t/openstack/build/d539b08e926049fa8ea297eda960b502 : FAILURE in 13m 48s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b4cbda4d02114849a30854bc8515497d : SUCCESS in 9m 30s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/1a4b96a411024158a82e91149e521951 : SUCCESS in 1h 09m 14s (non-voting)\n- nova-ceph-multistore https://zuul.opendev.org/t/openstack/build/00745c3c091344179641e24b9358f6b5 : SUCCESS in 1h 17m 56s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/967b26cf092a4d17a0a017338f712d67 : SUCCESS in 51m 02s\n- nova-live-migration-ceph https://zuul.opendev.org/t/openstack/build/6570ea5634924c02b9aafedb905b05d0 : SUCCESS in 54m 12s (non-voting)\n- nova-multi-cell https://zuul.opendev.org/t/openstack/build/3b3eb258d92b4dff9bcd8819e9c9bdcf : SUCCESS in 1h 34m 39s\n- nova-next https://zuul.opendev.org/t/openstack/build/a719cfd9a3544c84bb0098d6580e3708 : SUCCESS in 1h 44m 52s\n- nova-ovs-hybrid-plug https://zuul.opendev.org/t/openstack/build/c7a20c103f404c9a9b9a7a7f9c809327 : SUCCESS in 54m 31s\n- nova-tox-validate-backport https://zuul.opendev.org/t/openstack/build/8b1b7cd0c13d46ebb153a1b2a6aa5eff : SUCCESS in 5m 06s (non-voting)\n- nova-tox-functional-py38 https://zuul.opendev.org/t/openstack/build/ea17cacf95bf4e25b7e8ebcd04a30858 : SUCCESS in 21m 16s\n- nova-tox-functional-py39 https://zuul.opendev.org/t/openstack/build/803eb801cc894350b86387b0209ea61a : SUCCESS in 22m 06s\n- nova-tox-functional-py310 https://zuul.opendev.org/t/openstack/build/c3912e701fd5465ba786a0828ec198ad : FAILURE in 19m 56s (non-voting)\n- nova-grenade-multinode https://zuul.opendev.org/t/openstack/build/848fea01ed6f4e39a892a3bee2b32717 : SUCCESS in 1h 13m 05s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/917ae0bfb3844c3da9e1bc9ccb2e3edd : SUCCESS in 55m 41s\n- cyborg-tempest https://zuul.opendev.org/t/openstack/build/93e575a4fbdf4476a1a14b2d5ff900a5 : SUCCESS in 38m 24s (non-voting)\n- barbican-tempest-plugin-simple-crypto https://zuul.opendev.org/t/openstack/build/4b6074a4c0924fca844fd3bc4da37ef4 : SUCCESS in 41m 53s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"7782b38ce5d94b7a1e3ca884b3eed06da0e1b80b","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-02 11:54:18.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"b4a19b750aac710cdba9df38a5970a26de4c273d","author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"date":"2022-08-02 12:06:40.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"eaf10506135507fa4045093041aad52add982c4b","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 12:43:24.000000000","message":"Patch Set 2:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/9861f9174e8241ea987a68e57de9fb8c\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/065b346b4dbf4696960d5b414fae3cf8 : FAILURE in 46m 48s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/2a7eb67f419448a58385710b4b2f7505 : SUCCESS in 19m 22s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"cff07f36658cee9faceb2513a8aa693c301cf8fd","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-02 13:35:51.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"1603571a740cbd3df9cd04d0aa04bfba4ca6aef5","author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"date":"2022-08-02 13:39:26.000000000","message":"Patch Set 2: Code-Review-1\n\n(3 comments)","accounts_in_message":[],"_revision_number":2},{"id":"2cb2b68d5c41b3bbf4d012fd01c8119bc0c904c6","author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"date":"2022-08-02 13:44:53.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"6ecfbbb64b094bea82a65aa58950af8bec12d995","author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"date":"2022-08-02 13:45:20.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"275b480dcf513183ff4aea1b37f42f5008baace0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 13:46:10.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/2875b45d4a6f46ec99d608934cba157c\n\n- tempest-integrated-compute https://zuul.opendev.org/t/openstack/build/44b3b1fded044d979c71901bd98970e2 : SUCCESS in 1h 35m 36s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/2372336f8da942c59307aea0cab815c7 : SUCCESS in 55m 35s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/095b9c535fac4c6d889b8509ee3d3b96 : SUCCESS in 17m 47s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/0feebdae61814d3fb98739c46502f460 : SUCCESS in 13m 31s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/f4cfb78f43f445eeaac18030868ae819 : SUCCESS in 13m 13s\n- openstack-tox-py39 https://zuul.opendev.org/t/openstack/build/02f0a968e79e4059a3cffd210593c02e : SUCCESS in 13m 20s\n- openstack-tox-py310 https://zuul.opendev.org/t/openstack/build/a69e8231c31e4f41b9c1a1a6e3cc70a0 : FAILURE in 15m 04s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/85979b0696904a3697529209e1b95ff4 : SUCCESS in 11m 25s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/a8b455057d8d4b8983dc48bf3505676f : SUCCESS in 1h 18m 31s (non-voting)\n- nova-ceph-multistore https://zuul.opendev.org/t/openstack/build/2f9355c1295644798371d5f65ac37075 : SUCCESS in 1h 18m 32s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/879a6c781ba54646a6499a29a1a5d845 : SUCCESS in 58m 12s\n- nova-live-migration-ceph https://zuul.opendev.org/t/openstack/build/37f8cf62ea314f92903519eda5018e0d : SUCCESS in 1h 02m 51s (non-voting)\n- nova-multi-cell https://zuul.opendev.org/t/openstack/build/48f6414b8acf4a218ed99b280f8ab43f : SUCCESS in 1h 42m 54s\n- nova-next https://zuul.opendev.org/t/openstack/build/4a670ece11b74c16b6a4acdd7a17cec8 : SUCCESS in 1h 36m 54s\n- nova-ovs-hybrid-plug https://zuul.opendev.org/t/openstack/build/99b4cda1721342ddb4984d11e585b658 : SUCCESS in 1h 02m 03s\n- nova-tox-validate-backport https://zuul.opendev.org/t/openstack/build/3469519f939e42e7a2cfe35e7a71048e : SUCCESS in 7m 06s (non-voting)\n- nova-tox-functional-py38 https://zuul.opendev.org/t/openstack/build/14646f7f64954342b58892a5287ae155 : SUCCESS in 21m 43s\n- nova-tox-functional-py39 https://zuul.opendev.org/t/openstack/build/a70ff334facc4914bde81d54bf78ffc4 : SUCCESS in 22m 07s\n- nova-tox-functional-py310 https://zuul.opendev.org/t/openstack/build/2dd4b3afa2f24a668edc186d1c69fbd7 : FAILURE in 21m 19s (non-voting)\n- nova-grenade-multinode https://zuul.opendev.org/t/openstack/build/fc225fdcfb844a0c877990250b0de53d : SUCCESS in 1h 21m 24s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/94622c956d4043eb9ca26cd6fe2c8dde : SUCCESS in 1h 06m 28s\n- cyborg-tempest https://zuul.opendev.org/t/openstack/build/34cfdd083e5a4be7abd18909c536fef0 : SUCCESS in 26m 01s (non-voting)\n- barbican-tempest-plugin-simple-crypto https://zuul.opendev.org/t/openstack/build/1787ec91775044829bb56ba8713c69a7 : SUCCESS in 48m 37s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"ef43f169560cd08101dd4b5767fb416b02ce86e6","author":{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},"date":"2022-08-02 13:46:38.000000000","message":"Patch Set 2:\n\nBuild failed.  For information on how to proceed, see https://docs.openstack.org/infra/manual/developers.html\n\n- EMC_PowerFlex_NOVA https://elab-os-logsrv.delllabs.net/32/851832/2/check/EMC_PowerFlex_NOVA/4095ae9/EMC_PowerFlex_NOVA/None : NOT_REGISTERED\n\nLeave a comment with \u0027run-DellEMC PowerFlex CI\u0027 to trigger a recheck. For more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/DellEMC_PowerFlex_CI","accounts_in_message":[],"_revision_number":2},{"id":"b400d05c05a42fb851ed5e195ddf15a04d718e09","author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"date":"2022-08-02 13:47:45.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"e06557647418f4f468bac0d1ad817310da1bcb63","tag":"autogenerated:cloudbase-ci","author":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"date":"2022-08-02 14:12:42.000000000","message":"Patch Set 2:\n\nFor rechecking only on the Cloudbase Nova Hyper-V CI, add a review comment with run-Cloudbase Nova Hyper-V CI\n\n- nova http://cloudbase-ci.com/nova/851832/2/iscsi : FAILURE in 2h 16m 31s","accounts_in_message":[],"_revision_number":2},{"id":"1f9f1accd712a0330d1d0772a11da739c070cec6","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"},"date":"2022-08-02 14:22:41.000000000","message":"Patch Set 2:\n\nBuild FAILED. log path: ${LOG_PATH}\nLeave a comment with \u0027run-Lightbits CI\u0027 to trigger a recheck. \nFor more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/LightbitsLabs_CI","accounts_in_message":[],"_revision_number":2},{"id":"afef6b380dd8f7700407961e2d413741605e7d2c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-02 15:00:33.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"4e022941fda3eb5c25c84c2d19d6c9fcb51d7a0b","author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"date":"2022-08-02 15:40:40.000000000","message":"Patch Set 3: Code-Review-1\n\n(1 comment)","accounts_in_message":[],"_revision_number":3},{"id":"13ad44594dbd95a61efbccd02bed4d580bfca628","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 15:54:08.000000000","message":"Patch Set 3:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/600e9a85ddb940f0873dfa3ba4935650\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/30af4750238f4a2f9d4db91cff198576 : FAILURE in 47m 02s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/be6e310c779f4d7ea222622cd5ac6f35 : FAILURE in 41m 18s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"76b7d7f4e937a5d42a891b9fe6b317e71cc0ab5b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-02 17:08:33.000000000","message":"Patch Set 3: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/1d72d2abbc7248c98edc14ba9ee6277f\n\n- tempest-integrated-compute https://zuul.opendev.org/t/openstack/build/3657a60ad57043c39e00bc36f5e0ea93 : SUCCESS in 1h 12m 44s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/6936fed667884539b077a8e03d2c00bc : SUCCESS in 43m 47s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/c43fb80559944dc69a1704bdd5b8d286 : SUCCESS in 18m 18s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/f8339dc992b64ca08bd4689d38643a44 : SUCCESS in 12m 48s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/8a53b546a7f44ed28103e89bb00d719a : SUCCESS in 15m 37s\n- openstack-tox-py39 https://zuul.opendev.org/t/openstack/build/d2e85ebcc3c3495fb6f2ad66dcfa12ee : SUCCESS in 13m 05s\n- openstack-tox-py310 https://zuul.opendev.org/t/openstack/build/bd3098170ad749d1bcb5d102101cc64e : FAILURE in 12m 39s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b87197c02aa74f43810961e140be6a2b : SUCCESS in 10m 55s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/573f3c40195d4eceaa8523bff70f789d : SUCCESS in 1h 13m 51s (non-voting)\n- nova-ceph-multistore https://zuul.opendev.org/t/openstack/build/79bf2d2bb49645f2ad8ac1b4044b72d6 : SUCCESS in 1h 10m 50s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/844a9e71088243539546cf1bd9e151e6 : SUCCESS in 57m 42s\n- nova-live-migration-ceph https://zuul.opendev.org/t/openstack/build/735b221651354ee483e948ee8f8e913a : SUCCESS in 53m 20s (non-voting)\n- nova-multi-cell https://zuul.opendev.org/t/openstack/build/f6cce780954d46b28479d77662b84985 : SUCCESS in 1h 44m 28s\n- nova-next https://zuul.opendev.org/t/openstack/build/172f66c119f443e1afd0f90345600eb3 : SUCCESS in 1h 32m 47s\n- nova-ovs-hybrid-plug https://zuul.opendev.org/t/openstack/build/8c52f43b2d0c402b8417b0a8d599c9b3 : SUCCESS in 1h 01m 06s\n- nova-tox-validate-backport https://zuul.opendev.org/t/openstack/build/1cf225f779c447f6a4a2bd67675db6dc : SUCCESS in 6m 38s (non-voting)\n- nova-tox-functional-py38 https://zuul.opendev.org/t/openstack/build/7c8887cb4e254c67aa01bd3e819565eb : SUCCESS in 21m 34s\n- nova-tox-functional-py39 https://zuul.opendev.org/t/openstack/build/07f3b4768dd54453853b78840417ca25 : SUCCESS in 21m 36s\n- nova-tox-functional-py310 https://zuul.opendev.org/t/openstack/build/ae5bec57500d4915ae54ef9cbe5c0053 : FAILURE in 22m 40s (non-voting)\n- nova-grenade-multinode https://zuul.opendev.org/t/openstack/build/bfc9bc270b124e0ba59a08ee915e2e5e : SUCCESS in 1h 16m 21s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/5cc809c9b125457ba791f8a29d765a4d : SUCCESS in 1h 00m 40s\n- cyborg-tempest https://zuul.opendev.org/t/openstack/build/2a850371de2a43f68ff6674a5abeca21 : SUCCESS in 39m 08s (non-voting)\n- barbican-tempest-plugin-simple-crypto https://zuul.opendev.org/t/openstack/build/f7b32036c3374063bb73e10cbad356f8 : SUCCESS in 45m 58s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"1ad28b3c16263fdba97e060e9e9cf2208fd47ea6","author":{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},"date":"2022-08-02 17:08:57.000000000","message":"Patch Set 3:\n\nBuild failed.  For information on how to proceed, see https://docs.openstack.org/infra/manual/developers.html\n\n- EMC_PowerFlex_NOVA https://elab-os-logsrv.delllabs.net/32/851832/3/check/EMC_PowerFlex_NOVA/60fb196/EMC_PowerFlex_NOVA/None : NOT_REGISTERED\n\nLeave a comment with \u0027run-DellEMC PowerFlex CI\u0027 to trigger a recheck. For more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/DellEMC_PowerFlex_CI","accounts_in_message":[],"_revision_number":3},{"id":"4d32c36ffa34f4ac31773f1fef51de34dbebd74a","author":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"date":"2022-08-02 17:36:52.000000000","message":"Patch Set 3:\n\nBuild failed. Test completed on IBM PowerKVM platform. For rechecking only on the IBM PowerKVM CI, add a review comment with pkvm- recheck. For contact and more information, see https://wiki.openstack.org/wiki/PowerKVM\n\n- tempest-dsvm-full-focal-py3 finger://zuul-v3-executor.novalocal/41f4092d73124d2190b65c0fadac5c92 : POST_FAILURE in 2h 18m 54s","accounts_in_message":[],"_revision_number":3},{"id":"305903974d41c9222ebe94c8b5975d9c684e38c2","tag":"autogenerated:cloudbase-ci","author":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"date":"2022-08-02 17:37:54.000000000","message":"Patch Set 3:\n\nBuild succeeded. For checking Nova with Ceph, add a comment with run-Cloudbase Nova Ceph Hyper-V CI\n\n- nova http://cloudbase-ci.com/nova/851832/3/iscsi : SUCCESS in 2h 35m 17s","accounts_in_message":[],"_revision_number":3},{"id":"9e27c0e2333d5acc9f163767194a67b99c4dcccd","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"},"date":"2022-08-02 18:04:38.000000000","message":"Patch Set 3:\n\nBuild FAILED. log path: ${LOG_PATH}\nLeave a comment with \u0027run-Lightbits CI\u0027 to trigger a recheck. \nFor more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/LightbitsLabs_CI","accounts_in_message":[],"_revision_number":3},{"id":"db22971dbdf15a45d2eca11ece76f7effceddf05","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-03 08:43:18.000000000","message":"Patch Set 3:\n\n(1 comment)","accounts_in_message":[],"_revision_number":3},{"id":"93d31b932cbcf047f9e0b4f7fae99f4b2f197463","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-03 13:13:52.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"8e22b67a651e2d3e41fc800831c414678433005f","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-03 13:14:04.000000000","message":"Patch Set 3:\n\n(3 comments)","accounts_in_message":[],"_revision_number":3},{"id":"49d4205a3ea43283f77619c2ec195c1a1cbdd88d","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-03 13:47:19.000000000","message":"Patch Set 4:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/c1112f9faf8642bc8eee8581884bc26e\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/65f8641c4d2b4f429f658a861154c8b6 : SUCCESS in 31m 38s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/ec241771bd144c65a67a583a797556cd : SUCCESS in 30m 39s (non-voting)","accounts_in_message":[],"_revision_number":4},{"id":"f8af6eab50a734c810534b55d18d31c243cc2184","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2022-08-03 14:41:23.000000000","message":"Uploaded patch set 5: Patch Set 4 was rebased.","accounts_in_message":[],"_revision_number":5},{"id":"bf421881492bb35f4353fae01ff3288c3737310a","author":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"date":"2022-08-03 15:08:12.000000000","message":"Patch Set 5:\n\nBuild failed. Test completed on IBM PowerKVM platform. For rechecking only on the IBM PowerKVM CI, add a review comment with pkvm- recheck. For contact and more information, see https://wiki.openstack.org/wiki/PowerKVM\n\n- tempest-dsvm-full-focal-py3 tempest-dsvm-full-focal-py3 : NODE_FAILURE in 0s","accounts_in_message":[],"_revision_number":5},{"id":"bbf6bb053b8572b215975c9385da967f60e00936","author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"date":"2022-08-03 16:11:58.000000000","message":"Patch Set 5:\n\n(4 comments)","accounts_in_message":[],"_revision_number":5},{"id":"6878b7e2176d013a19f4634f631a9f482230e67f","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-03 16:34:51.000000000","message":"Patch Set 5:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/6dc9280863c14f1f983d06dbf0453c30\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/0c8f7775210c4cf5935db77fea0351ea : SUCCESS in 56m 21s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/076083e7994f4bb3bfd507c364957360 : SUCCESS in 37m 48s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"ad0208ecd8b8deffdfe431948d7ca2481c429bab","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-08-03 16:35:24.000000000","message":"Patch Set 5: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/5de4ac9e1e6a4f46bc093062c1741739\n\n- tempest-integrated-compute https://zuul.opendev.org/t/openstack/build/6581c795356d43b8bdc56376f999a401 : SUCCESS in 1h 30m 52s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/798eed9f8fd1437c9f82635cb1400d9e : SUCCESS in 52m 39s\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/0e67171b7ba3419189ad734e5e534deb : SUCCESS in 17m 49s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/6f8e85f69ab54eb19bdaedbd86d5b3de : SUCCESS in 11m 15s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/5de2103ffeda4394a5c12fde79166a7b : SUCCESS in 13m 23s\n- openstack-tox-py39 https://zuul.opendev.org/t/openstack/build/6e032ac86014481c97f976858cd68f23 : SUCCESS in 16m 00s\n- openstack-tox-py310 https://zuul.opendev.org/t/openstack/build/50a73dddf0e1445194ff3cf1274c23e3 : FAILURE in 18m 27s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e975a9fb9cf945a98874f0086f9bb3e6 : SUCCESS in 11m 33s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/337b4d5e6bd94588ab018e257ba55299 : SUCCESS in 1h 03m 31s (non-voting)\n- nova-ceph-multistore https://zuul.opendev.org/t/openstack/build/be76ca649ccb4cc7856782c44846c849 : SUCCESS in 1h 08m 10s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/a1e931342daf4e9b8b5801ba933f3a17 : SUCCESS in 49m 45s\n- nova-live-migration-ceph https://zuul.opendev.org/t/openstack/build/30cc9dda561e42f9901c93775f69278a : SUCCESS in 47m 16s (non-voting)\n- nova-lvm https://zuul.opendev.org/t/openstack/build/7d79e848fa4e4c798b88f6f07649529f : SUCCESS in 1h 06m 13s\n- nova-multi-cell https://zuul.opendev.org/t/openstack/build/b3bb8b820585431d8f02d4e07f92af54 : SUCCESS in 1h 34m 34s\n- nova-next https://zuul.opendev.org/t/openstack/build/26d24a5075844c7ba0ee59d638645c3e : SUCCESS in 1h 44m 40s\n- nova-ovs-hybrid-plug https://zuul.opendev.org/t/openstack/build/c25c391cbafe4ab4affd371795a30965 : SUCCESS in 1h 01m 46s\n- nova-tox-validate-backport https://zuul.opendev.org/t/openstack/build/ea879172bb804b44b17143cf47cf787b : SUCCESS in 9m 48s (non-voting)\n- nova-tox-functional-py38 https://zuul.opendev.org/t/openstack/build/888dc9d16f1c4694a2ef20f5c4b691c2 : SUCCESS in 23m 58s\n- nova-tox-functional-py39 https://zuul.opendev.org/t/openstack/build/4d1e6900b17c4c6096de2f30965a6ec1 : SUCCESS in 24m 56s\n- nova-tox-functional-py310 https://zuul.opendev.org/t/openstack/build/78ef0717c942486f9a6ed118a1dad898 : FAILURE in 24m 44s (non-voting)\n- nova-grenade-multinode https://zuul.opendev.org/t/openstack/build/f854859bc97b4ec7b464f2389bd90873 : SUCCESS in 1h 05m 30s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/0c5e0428c5d7430fa700bc80b52d5c21 : SUCCESS in 56m 24s\n- cyborg-tempest https://zuul.opendev.org/t/openstack/build/819ddd22e2d340c7b9411d78bff53538 : SUCCESS in 38m 04s (non-voting)\n- barbican-tempest-plugin-simple-crypto https://zuul.opendev.org/t/openstack/build/dc481fb875d049ccb0e78ccc5147c471 : SUCCESS in 50m 04s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"e2f91722810c615f2eb54b8f31da4a9bc7ed4a59","author":{"_account_id":15941,"name":"DellEMC PowerFlex CI","email":"emc.scaleio.ci@emc.com","username":"emc-scaleio-ci","tags":["SERVICE_USER"]},"date":"2022-08-03 16:35:59.000000000","message":"Patch Set 5:\n\nBuild failed.  For information on how to proceed, see https://docs.openstack.org/infra/manual/developers.html\n\n- EMC_PowerFlex_NOVA https://elab-os-logsrv.delllabs.net/32/851832/5/check/EMC_PowerFlex_NOVA/20acc6a/EMC_PowerFlex_NOVA/None : NOT_REGISTERED\n\nLeave a comment with \u0027run-DellEMC PowerFlex CI\u0027 to trigger a recheck. For more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/DellEMC_PowerFlex_CI","accounts_in_message":[],"_revision_number":5},{"id":"e6320f8f5db4cd4d35d9bf63bd5bcb84ca82c96c","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":34420,"name":"openstack lightbits","display_name":"Lightbits CI","email":"openstack@lightbitslabs.com","username":"openstack_lightbits"},"date":"2022-08-03 17:31:19.000000000","message":"Patch Set 5:\n\nBuild FAILED. log path: ${LOG_PATH}\nLeave a comment with \u0027run-Lightbits CI\u0027 to trigger a recheck. \nFor more information about CI, please see https://wiki.openstack.org/wiki/ThirdPartySystems/LightbitsLabs_CI","accounts_in_message":[],"_revision_number":5},{"id":"9368b4b11b3ab8ea3b9adc2d7d26f581cde51766","tag":"autogenerated:cloudbase-ci","author":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"date":"2022-08-03 23:54:01.000000000","message":"Patch Set 5:\n\nFor rechecking only on the Cloudbase Nova Hyper-V CI, add a review comment with run-Cloudbase Nova Hyper-V CI\n\n- nova http://cloudbase-ci.com/nova/851832/5/iscsi : FAILURE in 2h 45m 29s","accounts_in_message":[],"_revision_number":5}],"current_revision_number":5,"current_revision":"0b4107029650b687f012ab9519d68bd866349b54","revisions":{"b5e6a66bdc093b7ab91c2856513b915d14da0354":{"kind":"REWORK","_number":1,"created":"2022-08-02 09:35:43.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/32/851832/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/32/851832/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/32/851832/1"}}},"commit":{"parents":[{"commit":"788849ee6ca2d117211caca9cadcc14e93425f7a","subject":"Reproducer for bug 1982497","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/788849ee6ca2d117211caca9cadcc14e93425f7a"}]}],"author":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-01 16:01:41.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 09:34:23.000000000","tz":120},"subject":"Avoid saving instance under mutated migration context","message":"Avoid saving instance under mutated migration context\n\nDuring _rollback_live_migration there are two RPC message to the\ndestination host:\n\n* rollback_live_migration_at_destination is an RPC *cast* to clean up\n  the physically clean up the instance on the destination\n* drop_move_claim_at_destination is an RPC call to remove the resource\n  allocation of the instance from the dest\n\nThere is a bug in rollback_live_migration_at_destination. To be able to\nclean up the instance properly it temporarily applies the migration\ncontext to the instance then calls driver.destroy(). However that driver\ncall in case of libvirt driver eventually calls instance.save()\npersisting the instance pointing to the destination. This is not always\ncases a visible error as drop_move_claim_at_destination also persist the\ninstance, but with the right data, pointing to the source host. So if\nthe two RPC message handled in the order of\nrollback_live_migration_at_destination, drop_move_claim_at_destination\nthen the instance eventually get into a good state in the DB. BUT\nbecause rollback_live_migration_at_destination is an RPC cast, the\ncaller, the source host, does not wait for it to finish before\ndrop_move_claim_at_destination is called. Also\nrollback_live_migration_at_destination does IO (deletes instance disk)\nit can be delayed significantly causing an execution order of\ndrop_move_claim_at_destination then\nrollback_live_migration_at_destination. If this order happens then the\ninstance is left in an inconsistent state. In the DB it points and\nallocates from the destination node, while in reality it runs on the\nsource node. This can cause various resource allocation failures. The\none we observed happens when the instance allocates different pinned\nCPUs on the two hosts. In this case if the inconsistent instance is\ntried to be deleted the nova-compute on the source host tries to unpin\ncpus that are not pinned causing that the instance deletion fails.\n\nTo fix this we need to avoid the root cause to happen, namely the\ninstance.save() call during rollback_live_migration_at_destination\nwhile the instance in under the mutated migration context. The doc of\nthe Instance.mutated_migration_context context manager already state the\nit is dangerous to save the instance while this context is applied.\nHowever in the current case the context mutation and the instance.save\ncalls are very far from each other causing this hard to detect mistake.\nStill rollback_live_migration_at_destination needs the mutation to work\nproperly so our only option is to avoid the instance.save during\ndriver.destroy if it is called from rollback_live_migration_at_destination.\n\nOne can suggest to switch the rollback_live_migration_at_destination to\nbe an RPC call. However this would not totally remove the inconsistent\nstate, as rollback_live_migration_at_destination would still persist the\nwrong data to the DB, but only reduce the time window when such\ninconsistent state exists, and would make sure that the data is\neventually consistent. Also this would make the solution non\nbackportable due to the RPC bump requirement.\n\nSo the chosen solution is to skip instance.save() during\nrollback_live_migration_at_destination. Also we chose to do that via\nextending the mutated_migration_context context manage with an\nignore_save flag to trigger the skip. While this is less explicit than\npushing through such flag via the actual driver.destroy call, it results\nin a self contained solution that does not require a change in the virt\ndriver interface.\n\nSo what exactly we lose with the skipped instance.save() during\nrollback_live_migration_at_destination? Based on the libvirt driver code\nthe only thing that would be good to save on the instance are:\n* instance.system_metadata.clean_attempts\n* instance.cleaned\nThe rest of the changes on the instance is due to mutated migration\ncontext. The lost data is used by _run_pending_deletes() periodic. The\ngoal there is to retry failed local disk cleanups. The lost data only\nmeans that the periodic will try to clean this instance again one more\ntime unnecessarily. However the code handles the case when the instance\nis cleaned already. So the lost data does not cause any harm in this case.\n\nThis change fixes bug 1982497 fully and also fixes most of the problems\nreported in bug 1894095 but not all of them. In the latter the periodic\nrun still can re-create already freed allocation on the destination\nsomehow.\n\nTODO:\n* the solution does not work in a real RPC env as\n  mutated_migration_context is non remotable while save is so they are\n  operating on two different instance.\n\nCloses-Bug: #1982497\nRelated-Bug: #1894095\nChange-Id: I65534b52abfb1c6d331c73cc40413becdf3b05b1\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/b5e6a66bdc093b7ab91c2856513b915d14da0354"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/b5e6a66bdc093b7ab91c2856513b915d14da0354"}]},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"788849ee6ca2d117211caca9cadcc14e93425f7a","is_merged_in_target_branch":false,"change_id":"I08f2d6b54e2837e5918bf8a951794e93e4d80952","change_number":850672,"patch_set_number":2,"change_status":"NEW"}],"branch":"refs/heads/master"},"3ebb0ff5c5362581af93e48f23e7f5a2f2c829c4":{"kind":"REWORK","_number":2,"created":"2022-08-02 11:54:18.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/32/851832/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/32/851832/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/32/851832/2"}}},"commit":{"parents":[{"commit":"788849ee6ca2d117211caca9cadcc14e93425f7a","subject":"Reproducer for bug 1982497","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/788849ee6ca2d117211caca9cadcc14e93425f7a"}]}],"author":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-01 16:01:41.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 11:49:07.000000000","tz":120},"subject":"Avoid saving instance under mutated migration context","message":"Avoid saving instance under mutated migration context\n\nDuring _rollback_live_migration there are two RPC message to the\ndestination host:\n\n* rollback_live_migration_at_destination is an RPC *cast* to clean up\n  the physically clean up the instance on the destination\n* drop_move_claim_at_destination is an RPC call to remove the resource\n  allocation of the instance from the dest\n\nThere is a bug in rollback_live_migration_at_destination. To be able to\nclean up the instance properly it temporarily applies the migration\ncontext to the instance then calls driver.destroy(). However that driver\ncall in case of libvirt driver eventually calls instance.save()\npersisting the instance pointing to the destination. This is not always\ncases a visible error as drop_move_claim_at_destination also persist the\ninstance, but with the right data, pointing to the source host. So if\nthe two RPC message handled in the order of\nrollback_live_migration_at_destination, drop_move_claim_at_destination\nthen the instance eventually get into a good state in the DB. BUT\nbecause rollback_live_migration_at_destination is an RPC cast, the\ncaller, the source host, does not wait for it to finish before\ndrop_move_claim_at_destination is called. Also\nrollback_live_migration_at_destination does IO (deletes instance disk)\nit can be delayed significantly causing an execution order of\ndrop_move_claim_at_destination then\nrollback_live_migration_at_destination. If this order happens then the\ninstance is left in an inconsistent state. In the DB it points and\nallocates from the destination node, while in reality it runs on the\nsource node. This can cause various resource allocation failures. The\none we observed happens when the instance allocates different pinned\nCPUs on the two hosts. In this case if the inconsistent instance is\ntried to be deleted the nova-compute on the source host tries to unpin\ncpus that are not pinned causing that the instance deletion fails.\n\nTo fix this we need to avoid the root cause to happen, namely the\ninstance.save() call during rollback_live_migration_at_destination\nwhile the instance in under the mutated migration context. The doc of\nthe Instance.mutated_migration_context context manager already state the\nit is dangerous to save the instance while this context is applied.\nHowever in the current case the context mutation and the instance.save\ncalls are very far from each other causing this hard to detect mistake.\nStill rollback_live_migration_at_destination needs the mutation to work\nproperly so our only option is to avoid the instance.save during\ndriver.destroy if it is called from rollback_live_migration_at_destination.\n\nOne can suggest to switch the rollback_live_migration_at_destination to\nbe an RPC call. However this would not totally remove the inconsistent\nstate, as rollback_live_migration_at_destination would still persist the\nwrong data to the DB, but only reduce the time window when such\ninconsistent state exists, and would make sure that the data is\neventually consistent. Also this would make the solution non\nbackportable due to the RPC bump requirement.\n\nSo the chosen solution is to skip instance.save() during\nrollback_live_migration_at_destination. Also we chose to do that via\nextending the mutated_migration_context context manage with an\nignore_save flag to trigger the skip. While this is less explicit than\npushing through such flag via the actual driver.destroy call, it results\nin a self contained solution that does not require a change in the virt\ndriver interface.\n\nThe solution needs run some checks on the local side before the\ninstance.save call is remoted to the conductor. To be able to do this\nthe patch makes instance.save non-remotable that calls the remoteable\ninstance._remote_save method. This causes the Instance object hash\nchange but this change does not require a version bump as save still\nexists and callable on the remote end.\n\nSo what exactly we lose with the skipped instance.save() during\nrollback_live_migration_at_destination? Based on the libvirt driver code\nthe only thing that would be good to save on the instance are:\n* instance.system_metadata.clean_attempts\n* instance.cleaned\nThe rest of the changes on the instance is due to mutated migration\ncontext. The lost data is used by _run_pending_deletes() periodic. The\ngoal there is to retry failed local disk cleanups. The lost data only\nmeans that the periodic will try to clean this instance again one more\ntime unnecessarily. However the code handles the case when the instance\nis cleaned already. So the lost data does not cause any harm in this case.\n\nThis change fixes bug 1982497 fully and also fixes most of the problems\nreported in bug 1894095 but not all of them. In the latter the periodic\nrun still can re-create already freed allocation on the destination\nsomehow.\n\nCloses-Bug: #1982497\nRelated-Bug: #1894095\nChange-Id: I65534b52abfb1c6d331c73cc40413becdf3b05b1\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/3ebb0ff5c5362581af93e48f23e7f5a2f2c829c4"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/3ebb0ff5c5362581af93e48f23e7f5a2f2c829c4"}]},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"788849ee6ca2d117211caca9cadcc14e93425f7a","is_merged_in_target_branch":false,"change_id":"I08f2d6b54e2837e5918bf8a951794e93e4d80952","change_number":850672,"patch_set_number":2,"change_status":"NEW"}],"branch":"refs/heads/master"},"0d56d1cc085635be1c25a1b7287caa5effd80f09":{"kind":"REWORK","_number":3,"created":"2022-08-02 15:00:33.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/32/851832/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/32/851832/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/32/851832/3"}}},"commit":{"parents":[{"commit":"788849ee6ca2d117211caca9cadcc14e93425f7a","subject":"Reproducer for bug 1982497","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/788849ee6ca2d117211caca9cadcc14e93425f7a"}]}],"author":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 14:39:23.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 14:39:23.000000000","tz":120},"subject":"Do not mutate migration context for rollback_live_migration_at_destination","message":"Do not mutate migration context for rollback_live_migration_at_destination\n\nDuring _rollback_live_migration there are two RPC message to the\ndestination host:\n\n* rollback_live_migration_at_destination is an RPC *cast* to clean up\n  the physically clean up the instance on the destination\n* drop_move_claim_at_destination is an RPC call to remove the resource\n  allocation of the instance from the dest\n\nThere is a bug in rollback_live_migration_at_destination. It calls the\ndriver.destory() under a mutated migration context. Then the\ndriver._cleanup() will save the instance creating an inconsistent state\nin the DB as the instance now points to the destination host but it is\nactually running on the source host. This is not always cause a visible\nerror as drop_move_claim_at_destination also persist the\ninstance, but with the right data, pointing to the source host. So if\nthe two RPC messages handled in the order of\nrollback_live_migration_at_destination, drop_move_claim_at_destination\nthen the instance eventually get into a good state in the DB. BUT\nbecause rollback_live_migration_at_destination is an RPC cast, the\ncaller, the source host, does not wait for it to finish before\ndrop_move_claim_at_destination is called. Also\nrollback_live_migration_at_destination does IO (deletes instance disk)\nit can be delayed significantly causing an execution order of\ndrop_move_claim_at_destination then\nrollback_live_migration_at_destination. If this order happens then the\ninstance is left in an inconsistent state. In the DB it points and\nallocates from the destination node, while in reality it runs on the\nsource node. This can cause various resource allocation failures. The\none we observed happens when the instance allocates different pinned\nCPUs on the two hosts. In this case if the inconsistent instance is\ntried to be deleted the nova-compute on the source host tries to unpin\ncpus that are not pinned causing that the instance deletion fails.\n\nTo fix this bug this patch remove the migration context mutation from\nthe rollback_live_migration_at_destination code. There was a code\ncomment that such mutation needed to properly clean up vpmem resources.\n\n # NOTE(luyao): Apply migration_context temporarily since it\u0027s\n # on destination host, we rely on instance object to cleanup\n # specific resources like vpmem\n\nHowever I don\u0027t see any vpmem related cleanup in this code path neither\nnow nor in the original patches introduced the mutation. Also by\nremoving the mutation no vpmem specific unit of functional test fails.\nSo I conclude that this mutation is simply not necessary.\n\nThis change fixes bug 1982497 fully and also fixes most of the problems\nreported in bug 1894095 but not all of them. In the latter the periodic\nrun still re-creates the already freed allocation on the destination\nsomehow.\n\nCloses-Bug: #1982497\nRelated-Bug: #1894095\nChange-Id: I65534b52abfb1c6d331c73cc40413becdf3b05b1\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/0d56d1cc085635be1c25a1b7287caa5effd80f09"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/0d56d1cc085635be1c25a1b7287caa5effd80f09"}]},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"788849ee6ca2d117211caca9cadcc14e93425f7a","is_merged_in_target_branch":false,"change_id":"I08f2d6b54e2837e5918bf8a951794e93e4d80952","change_number":850672,"patch_set_number":2,"change_status":"NEW"}],"branch":"refs/heads/master"},"5408c9b732b4bf8e80ed4cf7c9f04965548b7d90":{"kind":"REWORK","_number":4,"created":"2022-08-03 13:13:52.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/32/851832/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/32/851832/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/32/851832/4"}}},"commit":{"parents":[{"commit":"788849ee6ca2d117211caca9cadcc14e93425f7a","subject":"Reproducer for bug 1982497","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/788849ee6ca2d117211caca9cadcc14e93425f7a"}]}],"author":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 14:39:23.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-03 13:13:06.000000000","tz":120},"subject":"Do not mutate migration context for rollback_live_migration_at_destination","message":"Do not mutate migration context for rollback_live_migration_at_destination\n\nDuring _rollback_live_migration there are two RPC message to the\ndestination host:\n\n* rollback_live_migration_at_destination is an RPC *cast* to clean up\n  the physically clean up the instance on the destination\n* drop_move_claim_at_destination is an RPC call to remove the resource\n  allocation of the instance from the dest\n\nThere is a bug in rollback_live_migration_at_destination. It calls the\ndriver.destory() under a mutated migration context. Then the\ndriver._cleanup() will save the instance creating an inconsistent state\nin the DB as the instance now points to the destination host but it is\nactually running on the source host. This is not always cause a visible\nerror as drop_move_claim_at_destination also persist the\ninstance, but with the right data, pointing to the source host. So if\nthe two RPC messages handled in the order of\nrollback_live_migration_at_destination, drop_move_claim_at_destination\nthen the instance eventually get into a good state in the DB. BUT\nbecause rollback_live_migration_at_destination is an RPC cast, the\ncaller, the source host, does not wait for it to finish before\ndrop_move_claim_at_destination is called. Also\nrollback_live_migration_at_destination does IO (deletes instance disk)\nit can be delayed significantly causing an execution order of\ndrop_move_claim_at_destination then\nrollback_live_migration_at_destination. If this order happens then the\ninstance is left in an inconsistent state. In the DB it points and\nallocates from the destination node, while in reality it runs on the\nsource node. This can cause various resource allocation failures. The\none we observed happens when the instance allocates different pinned\nCPUs on the two hosts. In this case if the inconsistent instance is\ntried to be deleted the nova-compute on the source host tries to unpin\ncpus that are not pinned causing that the instance deletion fails.\n\nTo fix this bug this patch removes the migration context mutation from\nthe rollback_live_migration_at_destination code. The\nrollback_live_migration_at_destination eventually calls driver._cleanup\nthat queriesthe vpmem devices from instance.resources and zeroes out\nthe physical devices on the hypervisor. This is relying on\ninstance.resources to point to the vpmems on the destination host. Hence\nmutation of the migration context was required. To solve this the\nlibvirt driver code is changed not to rely on instance.resources but\nrely on the libvirt domain configuration instead. That source of\ninformation is available during cleanup and does not affect by the\nmigration context.\n\nThis change fixes bug 1982497 fully and also fixes most of the problems\nreported in bug 1894095 but not all of them. In the latter the periodic\nrun still re-creates the already freed allocation on the destination\nsomehow.\n\nCloses-Bug: #1982497\nRelated-Bug: #1894095\nChange-Id: I65534b52abfb1c6d331c73cc40413becdf3b05b1\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/5408c9b732b4bf8e80ed4cf7c9f04965548b7d90"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/5408c9b732b4bf8e80ed4cf7c9f04965548b7d90"}]},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"788849ee6ca2d117211caca9cadcc14e93425f7a","is_merged_in_target_branch":false,"change_id":"I08f2d6b54e2837e5918bf8a951794e93e4d80952","change_number":850672,"patch_set_number":2,"change_status":"NEW"}],"branch":"refs/heads/master"},"0b4107029650b687f012ab9519d68bd866349b54":{"kind":"TRIVIAL_REBASE","_number":5,"created":"2022-08-03 14:41:23.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/32/851832/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/32/851832/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/32/851832/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/32/851832/5"}}},"commit":{"parents":[{"commit":"11710cd1c43d865ba5df7e02e62ea30595500aea","subject":"Reproducer for bug 1982497","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/11710cd1c43d865ba5df7e02e62ea30595500aea"}]}],"author":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-02 14:39:23.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"gibi@redhat.com","date":"2022-08-03 13:50:16.000000000","tz":120},"subject":"Do not mutate migration context for rollback_live_migration_at_destination","message":"Do not mutate migration context for rollback_live_migration_at_destination\n\nDuring _rollback_live_migration there are two RPC message to the\ndestination host:\n\n* rollback_live_migration_at_destination is an RPC *cast* to clean up\n  the physically clean up the instance on the destination\n* drop_move_claim_at_destination is an RPC call to remove the resource\n  allocation of the instance from the dest\n\nThere is a bug in rollback_live_migration_at_destination. It calls the\ndriver.destory() under a mutated migration context. Then the\ndriver._cleanup() will save the instance creating an inconsistent state\nin the DB as the instance now points to the destination host but it is\nactually running on the source host. This is not always cause a visible\nerror as drop_move_claim_at_destination also persist the\ninstance, but with the right data, pointing to the source host. So if\nthe two RPC messages handled in the order of\nrollback_live_migration_at_destination, drop_move_claim_at_destination\nthen the instance eventually get into a good state in the DB. BUT\nbecause rollback_live_migration_at_destination is an RPC cast, the\ncaller, the source host, does not wait for it to finish before\ndrop_move_claim_at_destination is called. Also\nrollback_live_migration_at_destination does IO (deletes instance disk)\nit can be delayed significantly causing an execution order of\ndrop_move_claim_at_destination then\nrollback_live_migration_at_destination. If this order happens then the\ninstance is left in an inconsistent state. In the DB it points and\nallocates from the destination node, while in reality it runs on the\nsource node. This can cause various resource allocation failures. The\none we observed happens when the instance allocates different pinned\nCPUs on the two hosts. In this case if the inconsistent instance is\ntried to be deleted the nova-compute on the source host tries to unpin\ncpus that are not pinned causing that the instance deletion fails.\n\nTo fix this bug this patch removes the migration context mutation from\nthe rollback_live_migration_at_destination code. The\nrollback_live_migration_at_destination eventually calls driver._cleanup\nthat queriesthe vpmem devices from instance.resources and zeroes out\nthe physical devices on the hypervisor. This is relying on\ninstance.resources to point to the vpmems on the destination host. Hence\nmutation of the migration context was required. To solve this the\nlibvirt driver code is changed not to rely on instance.resources but\nrely on the libvirt domain configuration instead. That source of\ninformation is available during cleanup and does not affect by the\nmigration context.\n\nThis change fixes bug 1982497 fully and also fixes most of the problems\nreported in bug 1894095 but not all of them. In the latter the periodic\nrun still re-creates the already freed allocation on the destination\nsomehow.\n\nCloses-Bug: #1982497\nRelated-Bug: #1894095\nChange-Id: I65534b52abfb1c6d331c73cc40413becdf3b05b1\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/0b4107029650b687f012ab9519d68bd866349b54"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/0b4107029650b687f012ab9519d68bd866349b54"}]},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"11710cd1c43d865ba5df7e02e62ea30595500aea","is_merged_in_target_branch":false,"change_id":"I08f2d6b54e2837e5918bf8a951794e93e4d80952","change_number":850672,"patch_set_number":3,"change_status":"NEW"}],"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"},{"label":"Review-Priority","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Review-Priority","description":"Review Priority","status":"NOT_APPLICABLE","is_legacy":false,"applicability_expression_result":{"fulfilled":false,"status":"FAIL"},"submittability_expression_result":{"expression":"is:true","fulfilled":true,"status":"NOT_EVALUATED","passing_atoms":[],"failing_atoms":[],"atom_explanations":{}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}