)]}'
{"nova/compute/api.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2999b4cc3bd2cd92f7e053e9924acb13e62f4b24","unresolved":false,"context_lines":[{"line_number":2623,"context_line":"        if not nova_context.is_cell_failure_sentinel(result[cell_uuid]):"},{"line_number":2624,"context_line":"            inst \u003d result[cell_uuid]"},{"line_number":2625,"context_line":"            self._save_user_id_in_instance_mapping(im, inst)"},{"line_number":2626,"context_line":"            LOG.debug(\u0027Found instance (instance_type_id: %s) in cell %s with \u0027"},{"line_number":2627,"context_line":"                      \u0027flavor: %s\u0027,"},{"line_number":2628,"context_line":"                      (inst.instance_type_id"},{"line_number":2629,"context_line":"                       if \u0027instance_type_id\u0027 in inst else \u0027n/a\u0027),"}],"source_content_type":"text/x-python","patch_set":4,"id":"3fa7e38b_2408158f","line":2626,"updated":"2019-12-11 22:55:25.000000000","message":"This is the instance after the resize is verified:\n\nDec 11 19:32:17.913979 ubuntu-bionic-vexxhost-sjc1-0013341398 devstack@n-api.service[17534]: DEBUG nova.compute.api [None req-3ac7a931-d4d2-4701-97c6-713244e58459 tempest-ServerActionsTestJSON-75810404 tempest-ServerActionsTestJSON-75810404] [instance: 8dcc1ead-3937-4be7-b96b-f8922a8d09c2] Found instance (instance_type_id: 12) in cell b5a441fc-a7dc-4906-8374-70ee8c3c7674(cell1) with flavor: Flavor(created_at\u003d2019-12-11T19:22:12Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002742\u0027,id\u003d11,is_public\u003dTrue,memory_mb\u003d64,name\u003d\u0027m1.nano\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1) {{(pid\u003d17535) _get_instance_from_cell /opt/stack/nova/nova/compute/api.py:2632}}\n\n\nNote that it shows instance_type_id\u003d12 but with flavor.id\u003d11.","commit_id":"0542a36770858feb6bbb2a67051e9c51ce5aeaf9"}],"nova/compute/manager.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2999b4cc3bd2cd92f7e053e9924acb13e62f4b24","unresolved":false,"context_lines":[{"line_number":5940,"context_line":"            # NOTE(mriedem): This is likely where storing old/new_flavor on"},{"line_number":5941,"context_line":"            # the MigrationContext would make this cleaner."},{"line_number":5942,"context_line":"            self._set_instance_info(instance, flavor)"},{"line_number":5943,"context_line":"            LOG.debug(\u0027Instance flavor updated to new flavor: %s; \u0027"},{"line_number":5944,"context_line":"                      \u0027instance_type_id: %s\u0027,"},{"line_number":5945,"context_line":"                      instance.flavor, instance.instance_type_id)"},{"line_number":5946,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"3fa7e38b_24dfb50c","line":5943,"updated":"2019-12-11 22:55:25.000000000","message":"This is showing up correctly:\n\nDec 11 19:32:03.149222 ubuntu-bionic-vexxhost-sjc1-0013341398 nova-compute[27542]: DEBUG nova.compute.manager [None req-aaeb1598-3212-4050-98d6-653e90792627 tempest-ServerActionsTestJSON-75810404 tempest-ServerActionsTestJSON-75810404] Instance flavor updated to new flavor: Flavor(created_at\u003d2019-12-11T19:22:17Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002784\u0027,id\u003d12,is_public\u003dTrue,memory_mb\u003d128,name\u003d\u0027m1.micro\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1); instance_type_id: 12 {{(pid\u003d27542) _finish_snapshot_based_resize_at_dest /opt/stack/nova/nova/compute/manager.py:5945}}\n\nThe expected flavor, 84, is being used.","commit_id":"0542a36770858feb6bbb2a67051e9c51ce5aeaf9"}],"nova/conductor/tasks/cross_cell_migrate.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b46a87b3a86a5eae57ff7902d7c9aecc1e851c17","unresolved":false,"context_lines":[{"line_number":1031,"context_line":"        This is similar to how ``confirm_resize`` works in the compute service"},{"line_number":1032,"context_line":"        for same-cell resize."},{"line_number":1033,"context_line":"        \"\"\""},{"line_number":1034,"context_line":"        LOG.debug(\u0027Updating migration and instance status in target cell DB.\u0027,"},{"line_number":1035,"context_line":"                  instance\u003dself.instance)"},{"line_number":1036,"context_line":"        # Complete the migration confirmation."},{"line_number":1037,"context_line":"        self.migration.status \u003d \u0027confirmed\u0027"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_23f81248","line":1034,"updated":"2019-12-11 17:22:40.000000000","message":"Hitting this at:\n\nDec 10 22:55:38.570247 ubuntu-bionic-rax-ord-0013324332 nova-conductor[26322]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-bb53ad64-6b8f-43cf-957b-47b56a7936c7 tempest-ServerActionsTestJSON-1515108172 tempest-ServerActionsTestJSON-1515108172] [instance: 19fbfa2a-09ab-4037-946e-e22182872c3a] Updating migration and instance status in target cell DB. {{(pid\u003d27687) _finish_confirm_in_target_cell /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:1035}}","commit_id":"62291e63c34201c56e5c634d8c30ae4c8fc7b6bd"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b46a87b3a86a5eae57ff7902d7c9aecc1e851c17","unresolved":false,"context_lines":[{"line_number":1049,"context_line":"        # _confirm_resize_on_deleting then the instance is being deleted."},{"line_number":1050,"context_line":"        self.instance.save(expected_task_state\u003d["},{"line_number":1051,"context_line":"            None, task_states.DELETING, task_states.SOFT_DELETING])"},{"line_number":1052,"context_line":"        LOG.debug(\u0027Updated instance in target cell DB: %s\u0027, self.instance,"},{"line_number":1053,"context_line":"                  instance\u003dself.instance)"},{"line_number":1054,"context_line":""},{"line_number":1055,"context_line":"    def _set_vm_and_task_state(self):"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_a399621e","line":1052,"updated":"2019-12-11 17:22:40.000000000","message":"Hitting this at:\n\nDec 10 22:55:38.652749 ubuntu-bionic-rax-ord-0013324332 nova-conductor[26322]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-bb53ad64-6b8f-43cf-957b-47b56a7936c7 tempest-ServerActionsTestJSON-1515108172 tempest-ServerActionsTestJSON-1515108172] [instance: 19fbfa2a-09ab-4037-946e-e22182872c3a] Updated instance in target cell DB: Instance(access_ip_v4\u003dNone,access_ip_v6\u003dNone,architecture\u003dNone,auto_disk_config\u003dFalse,availability_zone\u003d\u0027nova\u0027,cell_name\u003dNone,cleaned\u003dFalse,config_drive\u003d\u0027\u0027,created_at\u003d2019-12-10T22:55:11Z,default_ephemeral_device\u003dNone,default_swap_device\u003dNone,deleted\u003dFalse,deleted_at\u003dNone,device_metadata\u003d\u003c?\u003e,disable_terminate\u003dFalse,display_description\u003d\u0027tempest-ServerActionsTestJSON-server-307556264\u0027,display_name\u003d\u0027tempest-ServerActionsTestJSON-server-307556264\u0027,ec2_ids\u003d\u003c?\u003e,ephemeral_gb\u003d0,ephemeral_key_uuid\u003dNone,fault\u003d\u003c?\u003e,flavor\u003dFlavor(11),hidden\u003dFalse,host\u003d\u0027ubuntu-bionic-rax-ord-0013324332\u0027,hostname\u003d\u0027tempest-serveractionstestjson-server-307556264\u0027,id\u003d1,image_ref\u003d\u00273a2ca693-bd8d-43c7-9298-2a256110afd4\u0027,info_cache\u003dInstanceInfoCache,instance_type_id\u003d12,kernel_id\u003d\u0027\u0027,key_data\u003d\u0027ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF8m81OdOEULkxQuM0c2E6m33dsL0RfYJTXoHg8ojuhNGVbpUMqfPS5sf0yG7jNrHXPxUAs4EGaL8D9Xz4UkdM9vLPJ0Ec2cGpPqWCWZJ6W6hUzHX4FRF7nVWu7gYGaWEc6YeCwmtwUlwLrHhB0zWxFkOtrSeJmI1FX0cLzG0Z4/R3+KEMdwFztFvMF91F7Bre5SyMtMWKKZw+/zaT7/oNOHedtfhWlcr+elpQ4shFOeLfoJsB1KAPsVSpfrZMBAYroWCPSlBcs34Y4PEO0OXBYic+vfeQQ6d65naarYXOxueXSmE88gkNARh+6PHvb1X5yARN7TFKwBMwjo0SwGMl Generated-by-Nova\u0027,key_name\u003d\u0027tempest-keypair-1542474049\u0027,keypairs\u003d\u003c?\u003e,launch_index\u003d0,launched_at\u003d2019-12-10T22:55:35Z,launched_on\u003d\u0027ubuntu-bionic-rax-ord-0013324211\u0027,locked\u003dFalse,locked_by\u003dNone,memory_mb\u003d128,metadata\u003d{},migration_context\u003dNone,new_flavor\u003dNone,node\u003d\u0027ubuntu-bionic-rax-ord-0013324332\u0027,numa_topology\u003dNone,old_flavor\u003dNone,os_type\u003dNone,pci_devices\u003d\u003c?\u003e,pci_requests\u003d\u003c?\u003e,power_state\u003d1,progress\u003d0,project_id\u003d\u0027b668a2aa431c4640ba8188468a297e54\u0027,ramdisk_id\u003d\u0027\u0027,reservation_id\u003d\u0027r-9yppdw06\u0027,resources\u003d\u003c?\u003e,root_device_name\u003d\u0027/dev/vda\u0027,root_gb\u003d1,security_groups\u003dSecurityGroupList,services\u003d\u003c?\u003e,shutdown_terminate\u003dFalse,system_metadata\u003d{boot_roles\u003d\u0027member,reader\u0027,image_base_image_ref\u003d\u00273a2ca693-bd8d-43c7-9298-2a256110afd4\u0027,image_container_format\u003d\u0027bare\u0027,image_disk_format\u003d\u0027qcow2\u0027,image_hw_rng_model\u003d\u0027virtio\u0027,image_min_disk\u003d\u00271\u0027,image_min_ram\u003d\u00270\u0027,owner_project_name\u003d\u0027tempest-ServerActionsTestJSON-1515108172\u0027,owner_user_name\u003d\u0027tempest-ServerActionsTestJSON-1515108172\u0027},tags\u003d\u003c?\u003e,task_state\u003dNone,terminated_at\u003dNone,trusted_certs\u003d\u003c?\u003e,updated_at\u003d2019-12-10T22:55:36Z,user_data\u003d\u0027IyEvYmluL3NoCmVjaG8gIlByaW50aW5nIGNpcnJvcyB1c2VyIGF1dGhvcml6ZWQga2V5cyIKY2F0IH5jaXJyb3MvLnNzaC9hdXRob3JpemVkX2tleXMgfHwgdHJ1ZQo\u003d\u0027,user_id\u003d\u0027e8ef1ac9edc74c0da5c7062ca5ad4915\u0027,uuid\u003d19fbfa2a-09ab-4037-946e-e22182872c3a,vcpu_model\u003d\u003c?\u003e,vcpus\u003d1,vm_mode\u003dNone,vm_state\u003d\u0027active\u0027) {{(pid\u003d27687) _finish_confirm_in_target_cell /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:1053}}\n\nThis isn\u0027t very helpful though:\n\nflavor\u003dFlavor(11)\n\nNeed to figure out what this is:\n\ninstance_type_id\u003d12\n\nThe test is looking for flavorid 84:\n\ntesttools.matchers._impl.MismatchError: \u002784\u0027 !\u003d \u002742\u0027: server flavor is not same as flavor!\n\nLooks like this logs that flavor though:\n\nDec 10 22:55:31.894860 ubuntu-bionic-rax-ord-0013324332 nova-compute[27872]: DEBUG nova.virt.hardware [None req-8e98852c-eb6e-4559-a793-8367ace4e228 tempest-ServerActionsTestJSON-1515108172 tempest-ServerActionsTestJSON-1515108172] Getting desirable topologies for flavor Flavor(created_at\u003d2019-12-10T22:45:52Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002784\u0027,id\u003d12,is_public\u003dTrue,memory_mb\u003d128,name\u003d\u0027m1.micro\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1)\n\nAnd that says id\u003d12 and flavorid\u003d\u002784\u0027 so that\u0027s the one we want and it is on the target instance as expected.\n\nSo if we have updated the target cell instance at 22:55:38.652749.\n\nLooking at the tempest log:\n\n2019-12-10 22:55:39,748 3631 INFO     [tempest.common.waiters] State transition \"VERIFY_RESIZE/None\" \u003d\u003d\u003e \"ACTIVE/None\" after 2 second wait\n\nso tempest makes a GET request:\n\n2019-12-10 22:55:39,979 3631 INFO     [tempest.lib.common.rest_client] Request (ServerActionsTestJSON:test_resize_server_confirm): 200 GET https://10.210.224.66/compute/v2.1/servers/19fbfa2a-09ab-4037-946e-e22182872c3a\n\nand gets back this:\n\n2019-12-10 22:55:39,979 3631 DEBUG    [tempest.lib.common.rest_client] Request - Headers: {\u0027Content-Type\u0027: \u0027application/json\u0027, \u0027Accept\u0027: \u0027application/json\u0027, \u0027X-Auth-Token\u0027: \u0027\u003comitted\u003e\u0027}\n        Body: None\n    Response - Headers: {\u0027date\u0027: \u0027Tue, 10 Dec 2019 22:55:39 GMT\u0027, \u0027server\u0027: \u0027Apache/2.4.29 (Ubuntu)\u0027, \u0027content-length\u0027: \u00271672\u0027, \u0027content-type\u0027: \u0027application/json\u0027, \u0027openstack-api-version\u0027: \u0027compute 2.1\u0027, \u0027x-openstack-nova-api-version\u0027: \u00272.1\u0027, \u0027vary\u0027: \u0027OpenStack-API-Version,X-OpenStack-Nova-API-Version\u0027, \u0027x-openstack-request-id\u0027: \u0027req-bb8d2443-3f99-4eed-bd9a-3d02343ad8b1\u0027, \u0027x-compute-request-id\u0027: \u0027req-bb8d2443-3f99-4eed-bd9a-3d02343ad8b1\u0027, \u0027connection\u0027: \u0027close\u0027, \u0027status\u0027: \u0027200\u0027, \u0027content-location\u0027: \u0027https://10.210.224.66/compute/v2.1/servers/19fbfa2a-09ab-4037-946e-e22182872c3a\u0027}\n        Body: b\u0027{\"server\": {\"id\": \"19fbfa2a-09ab-4037-946e-e22182872c3a\", \"name\": \"tempest-ServerActionsTestJSON-server-307556264\", \"status\": \"ACTIVE\", \"tenant_id\": \"b668a2aa431c4640ba8188468a297e54\", \"user_id\": \"e8ef1ac9edc74c0da5c7062ca5ad4915\", \"metadata\": {}, \"hostId\": \"676fea239d5dad3e53407cb25fd7646765872acbe4299b4fd48766e2\", \"image\": {\"id\": \"3a2ca693-bd8d-43c7-9298-2a256110afd4\", \"links\": [{\"rel\": \"bookmark\", \"href\": \"https://10.210.224.66/compute/images/3a2ca693-bd8d-43c7-9298-2a256110afd4\"}]}, \"flavor\": {\"id\": \"42\", \"links\": [{\"rel\": \"bookmark\", \"href\": \"https://10.210.224.66/compute/flavors/42\"}]}, \"created\": \"2019-12-10T22:55:11Z\", \"updated\": \"2019-12-10T22:55:39Z\", \"addresses\": {\"tempest-ServerActionsTestJSON-701312637-network\": [{\"version\": 4, \"addr\": \"10.1.0.3\", \"OS-EXT-IPS:type\": \"fixed\", \"OS-EXT-IPS-MAC:mac_addr\": \"fa:16:3e:84:16:bc\"}, {\"version\": 4, \"addr\": \"172.24.5.64\", \"OS-EXT-IPS:type\": \"floating\", \"OS-EXT-IPS-MAC:mac_addr\": \"fa:16:3e:84:16:bc\"}]}, \"accessIPv4\": \"\", \"accessIPv6\": \"\", \"links\": [{\"rel\": \"self\", \"href\": \"https://10.210.224.66/compute/v2.1/servers/19fbfa2a-09ab-4037-946e-e22182872c3a\"}, {\"rel\": \"bookmark\", \"href\": \"https://10.210.224.66/compute/servers/19fbfa2a-09ab-4037-946e-e22182872c3a\"}], \"OS-DCF:diskConfig\": \"MANUAL\", \"progress\": 0, \"OS-EXT-AZ:availability_zone\": \"nova\", \"config_drive\": \"\", \"key_name\": \"tempest-keypair-1542474049\", \"OS-SRV-USG:launched_at\": \"2019-12-10T22:55:35.000000\", \"OS-SRV-USG:terminated_at\": null, \"security_groups\": [{\"name\": \"tempest-securitygroup--1194679413\"}], \"OS-EXT-STS:task_state\": null, \"OS-EXT-STS:vm_state\": \"active\", \"OS-EXT-STS:power_state\": 1, \"os-extended-volumes:volumes_attached\": []}}\u0027\n\nWhich shows:\n\n \"flavor\":{\n         \"id\":\"42\",\n         \"links\":[\n            {\n               \"rel\":\"bookmark\",\n               \"href\":\"https://10.210.224.66/compute/flavors/42\"\n            }\n         ]\n      },\n\nLooking at the API logs at that GET request:\n\nDec 10 22:55:39.767877 ubuntu-bionic-rax-ord-0013324332 devstack@n-api.service[17855]: DEBUG nova.compute.api [None req-bb8d2443-3f99-4eed-bd9a-3d02343ad8b1 tempest-ServerActionsTestJSON-1515108172 tempest-ServerActionsTestJSON-1515108172] [instance: 19fbfa2a-09ab-4037-946e-e22182872c3a] Getting instance from cell: 58203e6a-6306-4228-b7c5-3865b85ddbcc(cell1) {{(pid\u003d17857) _get_instance /opt/stack/nova/nova/compute/api.py:2675}}\n\nDec 10 22:55:39.816580 ubuntu-bionic-rax-ord-0013324332 devstack@n-api.service[17855]: DEBUG nova.compute.api [None req-bb8d2443-3f99-4eed-bd9a-3d02343ad8b1 tempest-ServerActionsTestJSON-1515108172 tempest-ServerActionsTestJSON-1515108172] [instance: 19fbfa2a-09ab-4037-946e-e22182872c3a] Found instance in cell 58203e6a-6306-4228-b7c5-3865b85ddbcc(cell1) with flavor: Flavor(created_at\u003d2019-12-10T22:45:48Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002742\u0027,id\u003d11,is_public\u003dTrue,memory_mb\u003d64,name\u003d\u0027m1.nano\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1) {{(pid\u003d17857) _get_instance_from_cell /opt/stack/nova/nova/compute/api.py:2629}}\n\nAnd that shows flavorid\u003d42.\n\nSo are we getting the instance from the correct cell? It pulled it from cell1. According to the conductor logs:\n\nDeleting instance record from source cell fe4b6eb4-a13f-4d54-a4f2-799b4aa475eb\n\nAnd the source cell fe4b6eb4-a13f-4d54-a4f2-799b4aa475eb is cell2:\n\n2019-12-10 22:44:14.455 | + lib/nova:start_nova:1060                 :   /usr/local/bin/nova-manage cell_v2 list_cells --verbose\n\n2019-12-10 22:44:18.051 | +-------+--------------------------------------+-----------------------------------------------------------------+-----------------------------------------------------------------------+----------+\n\n2019-12-10 22:44:18.051 | |  Name |                 UUID                 |                          Transport URL                          |                          Database Connection                          | Disabled |\n\n2019-12-10 22:44:18.051 | +-------+--------------------------------------+-----------------------------------------------------------------+-----------------------------------------------------------------------+----------+\n\n2019-12-10 22:44:18.051 | | cell0 | 00000000-0000-0000-0000-000000000000 |                             none:///                            | mysql+pymysql://root:secretdatabase@127.0.0.1/nova_cell0?charset\u003dutf8 |  False   |\n\n2019-12-10 22:44:18.051 | | cell1 | 58203e6a-6306-4228-b7c5-3865b85ddbcc | rabbit://stackrabbit:secretrabbit@10.210.224.66:5672/nova_cell1 | mysql+pymysql://root:secretdatabase@127.0.0.1/nova_cell1?charset\u003dutf8 |  False   |\n\n2019-12-10 22:44:18.051 | | cell2 | fe4b6eb4-a13f-4d54-a4f2-799b4aa475eb | rabbit://stackrabbit:secretrabbit@10.210.224.66:5672/nova_cell2 | mysql+pymysql://root:secretdatabase@127.0.0.1/nova_cell2?charset\u003dutf8 |  False   |\n\n\nSo wtf? The API is properly pulling the instance from cell1 and before it does we have updated the instance record in the cell1 database and from above we see the flavor is 84 but when the API pulls the instance the flavor is 42.\n\nWhat would be changing the instance.flavor in cell1 between when the ConfirmResizeTask changes the instance status and when the API pulls the instance? Is there a cache somewhere? Maybe need to add some logging to the Instance object when the flavor changes.","commit_id":"62291e63c34201c56e5c634d8c30ae4c8fc7b6bd"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2999b4cc3bd2cd92f7e053e9924acb13e62f4b24","unresolved":false,"context_lines":[{"line_number":1056,"context_line":"            None, task_states.DELETING, task_states.SOFT_DELETING])"},{"line_number":1057,"context_line":"        LOG.debug(\u0027Updated instance in target cell %s: %s - flavor: %s\u0027,"},{"line_number":1058,"context_line":"                  self.instance._context.cell_uuid, self.instance,"},{"line_number":1059,"context_line":"                  (self.instance.flavor"},{"line_number":1060,"context_line":"                   if \u0027flavor\u0027 in self.instance else \u0027n/a\u0027),"},{"line_number":1061,"context_line":"                  instance\u003dself.instance)"},{"line_number":1062,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"3fa7e38b_24ba55bf","line":1059,"updated":"2019-12-11 22:55:25.000000000","message":"OK this time when it\u0027s logged it shows that the cell stuff is all correct (nothing getting wonked with copying the context) but the flavor is wrong - it\u0027s 42 rather than 84:\n\nDec 11 19:32:16.351875 ubuntu-bionic-vexxhost-sjc1-0013341398 nova-conductor[26000]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-87819295-49b7-4ae6-9613-08e1fab29f4a tempest-ServerActionsTestJSON-75810404 tempest-ServerActionsTestJSON-75810404] [instance: 8dcc1ead-3937-4be7-b96b-f8922a8d09c2] Updated instance in target cell b5a441fc-a7dc-4906-8374-70ee8c3c7674: Instance(access_ip_v4\u003dNone,access_ip_v6\u003dNone,architecture\u003dNone,auto_disk_config\u003dFalse,availability_zone\u003d\u0027nova\u0027,cell_name\u003dNone,cleaned\u003dFalse,config_drive\u003d\u0027\u0027,created_at\u003d2019-12-11T19:31:39Z,default_ephemeral_device\u003dNone,default_swap_device\u003dNone,deleted\u003dFalse,deleted_at\u003dNone,device_metadata\u003d\u003c?\u003e,disable_terminate\u003dFalse,display_description\u003d\u0027tempest-ServerActionsTestJSON-server-1887183906\u0027,display_name\u003d\u0027tempest-ServerActionsTestJSON-server-1887183906\u0027,ec2_ids\u003d\u003c?\u003e,ephemeral_gb\u003d0,ephemeral_key_uuid\u003dNone,fault\u003d\u003c?\u003e,flavor\u003dFlavor(11),hidden\u003dFalse,host\u003d\u0027ubuntu-bionic-vexxhost-sjc1-0013341398\u0027,hostname\u003d\u0027tempest-serveractionstestjson-server-1887183906\u0027,id\u003d1,image_ref\u003d\u0027962bdf6a-05f0-4e12-b289-a9b77d8d1f90\u0027,info_cache\u003dInstanceInfoCache,instance_type_id\u003d12,kernel_id\u003d\u0027\u0027,key_data\u003d\u0027ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmrrlkZ3cONRrQcjsY7uVvIH2CJY47XkwOUBnPHtmu0mxacbZGP5a2UjeCrTElfMbaWG5wp+h3Y9axCVjmYsRojWjSJ798WJNCOY2+jfDtmnc5cJB4f+s4SomlZ8bKuhHrt3dO3hT+/OxJ0YOHWR7SIWEZJkiri7Wqgn7pbY8NEvLw20T2wWWXkkOuyfz6wiIs+zW+KArXnCTxGZwdh3pEtDniFSti1DUrjuVyaAeqQKMtZ6Xx1MXTiouaUM3eZUwt+HvnRZhRs2xUNBeorK18r6nDA0YvWmpHsu7AeZPcn2Zu+7c1QsHFBhcMI2qMhuEABe6XECGTSr4dorj59w0p Generated-by-Nova\u0027,key_name\u003d\u0027tempest-keypair-296263661\u0027,keypairs\u003d\u003c?\u003e,launch_index\u003d0,launched_at\u003d2019-12-11T19:32:12Z,launched_on\u003d\u0027ubuntu-bionic-vexxhost-sjc1-0013341399\u0027,locked\u003dFalse,locked_by\u003dNone,memory_mb\u003d128,metadata\u003d{},migration_context\u003dNone,new_flavor\u003dNone,node\u003d\u0027ubuntu-bionic-vexxhost-sjc1-0013341398\u0027,numa_topology\u003dNone,old_flavor\u003dNone,os_type\u003dNone,pci_devices\u003d\u003c?\u003e,pci_requests\u003d\u003c?\u003e,power_state\u003d1,progress\u003d0,project_id\u003d\u00276ee5e88d4892422ba33305aa5dcec8fd\u0027,ramdisk_id\u003d\u0027\u0027,reservation_id\u003d\u0027r-vbm5c6n5\u0027,resources\u003d\u003c?\u003e,root_device_name\u003d\u0027/dev/vda\u0027,root_gb\u003d1,security_groups\u003dSecurityGroupList,services\u003d\u003c?\u003e,shutdown_terminate\u003dFalse,system_metadata\u003d{boot_roles\u003d\u0027reader,member\u0027,image_base_image_ref\u003d\u0027962bdf6a-05f0-4e12-b289-a9b77d8d1f90\u0027,image_container_format\u003d\u0027bare\u0027,image_disk_format\u003d\u0027qcow2\u0027,image_hw_rng_model\u003d\u0027virtio\u0027,image_min_disk\u003d\u00271\u0027,image_min_ram\u003d\u00270\u0027,owner_project_name\u003d\u0027tempest-ServerActionsTestJSON-75810404\u0027,owner_user_name\u003d\u0027tempest-ServerActionsTestJSON-75810404\u0027},tags\u003d\u003c?\u003e,task_state\u003dNone,terminated_at\u003dNone,trusted_certs\u003d\u003c?\u003e,updated_at\u003d2019-12-11T19:32:13Z,user_data\u003d\u0027IyEvYmluL3NoCmVjaG8gIlByaW50aW5nIGNpcnJvcyB1c2VyIGF1dGhvcml6ZWQga2V5cyIKY2F0IH5jaXJyb3MvLnNzaC9hdXRob3JpemVkX2tleXMgfHwgdHJ1ZQo\u003d\u0027,user_id\u003d\u0027c9a42a5d5daa4cbd8e12eabc0c219311\u0027,uuid\u003d8dcc1ead-3937-4be7-b96b-f8922a8d09c2,vcpu_model\u003d\u003c?\u003e,vcpus\u003d1,vm_mode\u003dNone,vm_state\u003d\u0027active\u0027) - flavor: Flavor(created_at\u003d2019-12-11T19:22:12Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002742\u0027,id\u003d11,is_public\u003dTrue,memory_mb\u003d64,name\u003d\u0027m1.nano\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1) {{(pid\u003d27187) _finish_confirm_in_target_cell /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:1061}}\n\nSo between FinishResizeAtDestTask and ConfirmResizeTask we must be overwriting the flavor in the target cell instance? Maybe missing an instance.refresh() in FinishResizeAtDestTask?","commit_id":"0542a36770858feb6bbb2a67051e9c51ce5aeaf9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"164da4e3fd63177d7187de36fce3f84af04bb786","unresolved":false,"context_lines":[{"line_number":568,"context_line":"        # this exact moment only one copy of the instance will be returned."},{"line_number":569,"context_line":"        # DEBUG: finish_snapshot_based_resize_at_dest updated fields on the"},{"line_number":570,"context_line":"        # instance in the target cell db, see if we have a stale copy"},{"line_number":571,"context_line":"        LOG.debug(\u0027Target cell instance flavor after \u0027"},{"line_number":572,"context_line":"                  \u0027finish_snapshot_based_resize_at_dest but before updating \u0027"},{"line_number":573,"context_line":"                  \u0027hidden\u003dFalse: %s\u0027, self.instance.flavor)"},{"line_number":574,"context_line":"        self.instance.hidden \u003d False"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_5fa15a39","line":571,"updated":"2019-12-12 01:21:21.000000000","message":"Dec 12 00:02:42.628793 ubuntu-bionic-rax-dfw-0013344366 nova-conductor[26468]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-22d60f43-8524-4d75-aa24-2bec74b96fec None None] Target cell instance flavor after finish_snapshot_based_resize_at_dest but before updating hidden\u003dFalse: Flavor(created_at\u003d2019-12-11T23:52:22Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002742\u0027,id\u003d11,is_public\u003dTrue,memory_mb\u003d64,name\u003d\u0027m1.nano\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1) {{(pid\u003d27811) _update_instance_mapping /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:573}}\n\nConfirmed that we have a stale copy here, it\u0027s using flavorid\u003d42.","commit_id":"d7783bc629160cf085d39e07aeb64c618a8f87f3"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"164da4e3fd63177d7187de36fce3f84af04bb786","unresolved":false,"context_lines":[{"line_number":572,"context_line":"                  \u0027finish_snapshot_based_resize_at_dest but before updating \u0027"},{"line_number":573,"context_line":"                  \u0027hidden\u003dFalse: %s\u0027, self.instance.flavor)"},{"line_number":574,"context_line":"        self.instance.hidden \u003d False"},{"line_number":575,"context_line":"        LOG.debug(\u0027Target cell instance obj_what_changed before saving \u0027"},{"line_number":576,"context_line":"                  \u0027hidden\u003dFalse: %s\u0027, self.instance.obj_what_changed())"},{"line_number":577,"context_line":"        self.instance.save()"},{"line_number":578,"context_line":"        LOG.debug(\u0027Target cell instance flavor after updating \u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_1f65221d","line":575,"updated":"2019-12-12 01:21:21.000000000","message":"And flavor is in here so save() is resetting it:\n\nDec 12 00:02:42.630470 ubuntu-bionic-rax-dfw-0013344366 nova-conductor[26468]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-22d60f43-8524-4d75-aa24-2bec74b96fec None None] Target cell instance obj_what_changed before saving hidden\u003dFalse: {\u0027ec2_ids\u0027, \u0027flavor\u0027, \u0027hidden\u0027, \u0027new_flavor\u0027} {{(pid\u003d27811) _update_instance_mapping /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:576}}","commit_id":"d7783bc629160cf085d39e07aeb64c618a8f87f3"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"164da4e3fd63177d7187de36fce3f84af04bb786","unresolved":false,"context_lines":[{"line_number":575,"context_line":"        LOG.debug(\u0027Target cell instance obj_what_changed before saving \u0027"},{"line_number":576,"context_line":"                  \u0027hidden\u003dFalse: %s\u0027, self.instance.obj_what_changed())"},{"line_number":577,"context_line":"        self.instance.save()"},{"line_number":578,"context_line":"        LOG.debug(\u0027Target cell instance flavor after updating \u0027"},{"line_number":579,"context_line":"                  \u0027hidden\u003dFalse: %s\u0027, self.instance.flavor)"},{"line_number":580,"context_line":"        if (\u0027migration_type\u0027 in self.migration and"},{"line_number":581,"context_line":"                self.migration.migration_type \u003d\u003d \u0027resize\u0027):"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_bf672e24","line":578,"updated":"2019-12-12 01:21:21.000000000","message":"Dec 12 00:02:42.700550 ubuntu-bionic-rax-dfw-0013344366 nova-conductor[26468]: DEBUG nova.conductor.tasks.cross_cell_migrate [None req-22d60f43-8524-4d75-aa24-2bec74b96fec None None] Target cell instance flavor after updating hidden\u003dFalse: Flavor(created_at\u003d2019-12-11T23:52:22Z,deleted\u003dFalse,deleted_at\u003dNone,description\u003dNone,disabled\u003dFalse,ephemeral_gb\u003d0,extra_specs\u003d{hw_rng:allowed\u003d\u0027True\u0027},flavorid\u003d\u002742\u0027,id\u003d11,is_public\u003dTrue,memory_mb\u003d64,name\u003d\u0027m1.nano\u0027,projects\u003d\u003c?\u003e,root_gb\u003d1,rxtx_factor\u003d1.0,swap\u003d0,updated_at\u003dNone,vcpu_weight\u003d0,vcpus\u003d1) {{(pid\u003d27811) _update_instance_mapping /opt/stack/nova/nova/conductor/tasks/cross_cell_migrate.py:579}}\n\nsame - this is why it\u0027s persisted as flavorid\u003d42.","commit_id":"d7783bc629160cf085d39e07aeb64c618a8f87f3"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"164da4e3fd63177d7187de36fce3f84af04bb786","unresolved":false,"context_lines":[{"line_number":582,"context_line":"            # The instance flavor values should be set and the flavor should"},{"line_number":583,"context_line":"            # be set to the new_flavor."},{"line_number":584,"context_line":"            if not self.instance.old_flavor:"},{"line_number":585,"context_line":"                LOG.error(\u0027instance old_flavor is not set in target cell\u0027)"},{"line_number":586,"context_line":"            if not self.instance.new_flavor:"},{"line_number":587,"context_line":"                LOG.error(\u0027instance new_flavor is not set in target cell\u0027)"},{"line_number":588,"context_line":"            if self.instance.instance_type_id !\u003d self.instance.flavor.id:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_9f7032dc","line":585,"updated":"2019-12-12 01:21:21.000000000","message":"old_flavor isn\u0027t set:\n\nDec 12 00:02:42.700960 ubuntu-bionic-rax-dfw-0013344366 nova-conductor[26468]: ERROR nova.conductor.tasks.cross_cell_migrate [None req-22d60f43-8524-4d75-aa24-2bec74b96fec None None] instance old_flavor is not set in target cell","commit_id":"d7783bc629160cf085d39e07aeb64c618a8f87f3"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"164da4e3fd63177d7187de36fce3f84af04bb786","unresolved":false,"context_lines":[{"line_number":586,"context_line":"            if not self.instance.new_flavor:"},{"line_number":587,"context_line":"                LOG.error(\u0027instance new_flavor is not set in target cell\u0027)"},{"line_number":588,"context_line":"            if self.instance.instance_type_id !\u003d self.instance.flavor.id:"},{"line_number":589,"context_line":"                LOG.error(\u0027instance_type_id (%d) !\u003d instance.flavor.id (%d)\u0027,"},{"line_number":590,"context_line":"                          self.instance.instance_type_id,"},{"line_number":591,"context_line":"                          self.instance.flavor.id)"},{"line_number":592,"context_line":"        # Update the instance mapping to point at the target cell. This is so"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_ff4a8680","line":589,"updated":"2019-12-12 01:21:21.000000000","message":"And the instance_type_id doesn\u0027t match the flavor:\n\nDec 12 00:02:42.701362 ubuntu-bionic-rax-dfw-0013344366 nova-conductor[26468]: ERROR nova.conductor.tasks.cross_cell_migrate [None req-22d60f43-8524-4d75-aa24-2bec74b96fec None None] instance_type_id (12) !\u003d instance.flavor.id (11)","commit_id":"d7783bc629160cf085d39e07aeb64c618a8f87f3"}]}