)]}'
{"nova/compute/manager.py":[{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"068738be086fc0cccab1dbbb8959cf73828f90db","unresolved":false,"context_lines":[{"line_number":6485,"context_line":"            # This is a bit weird, but we need to clear sys.exc_info() so that"},{"line_number":6486,"context_line":"            # oslo.log formatting does not inadvertently use it later if an"},{"line_number":6487,"context_line":"            # error message is logged without an explicit exc_info. This is"},{"line_number":6488,"context_line":"            # only a problem with python 2."},{"line_number":6489,"context_line":"            if six.PY2:"},{"line_number":6490,"context_line":"                sys.exc_clear()"},{"line_number":6491,"context_line":"        except exception.VirtualInterfacePlugException:"}],"source_content_type":"text/x-python","patch_set":1,"id":"5fc1f717_ddc61b17","line":6488,"updated":"2019-04-05 14:23:28.000000000","message":"I print log at here, I can see the bdms as empty list.\n\nhttp://logs.openstack.org/07/649807/2/check/nova-grenade-live-migration/0f97996/logs/screen-n-cpu.txt.gz#_Apr_04_14_15_25_470994\n\nThen I think this function return","commit_id":"36f9a385666d36d7591b11d478f9c1cf278bd3ce"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"068738be086fc0cccab1dbbb8959cf73828f90db","unresolved":false,"context_lines":[{"line_number":6532,"context_line":"        self._do_pre_live_migration_from_source("},{"line_number":6533,"context_line":"            context, dest, instance, block_migration, migration, migrate_data,"},{"line_number":6534,"context_line":"            source_bdms)"},{"line_number":6535,"context_line":""},{"line_number":6536,"context_line":"        # Set migrate_data.migration because that is how _post_live_migration"},{"line_number":6537,"context_line":"        # and _rollback_live_migration get the migration object for cleanup."},{"line_number":6538,"context_line":"        # Yes this is gross but changing the _post_live_migration and"}],"source_content_type":"text/x-python","patch_set":1,"id":"5fc1f717_1de02397","line":6535,"updated":"2019-04-05 14:23:28.000000000","message":"Just after the return, the bdms value is disappeared.\n\nhttp://logs.openstack.org/07/649807/2/check/nova-grenade-live-migration/0f97996/logs/screen-n-cpu.txt.gz#_Apr_04_14_15_25_471815\n\nI can\u0027t find out a reason to explain that.","commit_id":"36f9a385666d36d7591b11d478f9c1cf278bd3ce"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"d38a57c38a68dd2b427fe21e02b280ce72667861","unresolved":false,"context_lines":[{"line_number":1819,"context_line":"                                        bdms\u003dNone):"},{"line_number":1820,"context_line":"        \"\"\"Transform block devices to the driver block_device format.\"\"\""},{"line_number":1821,"context_line":""},{"line_number":1822,"context_line":"        if bdms is None:"},{"line_number":1823,"context_line":"            bdms \u003d objects.BlockDeviceMappingList.get_by_instance_uuid("},{"line_number":1824,"context_line":"                    context, instance.uuid)"},{"line_number":1825,"context_line":"        block_device_info \u003d driver.get_block_device_info(instance, bdms)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_3a4c5b9d","line":1822,"updated":"2019-09-18 17:48:08.000000000","message":"This checks specifically for None, but what if we get an empty list here? I kinda wanna try changing this to:\n\n  if not bdms:","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"9e1d7dc385233f22382df83c38732c0e06cc798d","unresolved":false,"context_lines":[{"line_number":1819,"context_line":"                                        bdms\u003dNone):"},{"line_number":1820,"context_line":"        \"\"\"Transform block devices to the driver block_device format.\"\"\""},{"line_number":1821,"context_line":""},{"line_number":1822,"context_line":"        if bdms is None:"},{"line_number":1823,"context_line":"            bdms \u003d objects.BlockDeviceMappingList.get_by_instance_uuid("},{"line_number":1824,"context_line":"                    context, instance.uuid)"},{"line_number":1825,"context_line":"        block_device_info \u003d driver.get_block_device_info(instance, bdms)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_1d0e3934","line":1822,"in_reply_to":"3fa7e38b_3a4c5b9d","updated":"2019-09-18 17:59:52.000000000","message":"I think I\u0027ve thought about doing that for different reasons in the past for different issues but can\u0027t remember what they were at this point, but I think the check for None is to tell the difference between (1) did the caller already check for bdms and there aren\u0027t so don\u0027t re-query and (2) the caller didn\u0027t check so it wants us to do the work for them.\n\nIn reality I don\u0027t think it\u0027s possible to not have any BDMs because even a non-volume-backed server is going to have a BDM with source_type\u003d\u0027image\u0027 and destination_type\u003d\u0027local\u0027.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"d38a57c38a68dd2b427fe21e02b280ce72667861","unresolved":false,"context_lines":[{"line_number":6863,"context_line":"        try:"},{"line_number":6864,"context_line":"            if (\u0027block_migration\u0027 in migrate_data and"},{"line_number":6865,"context_line":"                    migrate_data.block_migration):"},{"line_number":6866,"context_line":"                block_device_info \u003d self._get_instance_block_device_info("},{"line_number":6867,"context_line":"                    context, instance, bdms\u003dsource_bdms)"},{"line_number":6868,"context_line":"                disk \u003d self.driver.get_instance_disk_info("},{"line_number":6869,"context_line":"                    instance, block_device_info\u003dblock_device_info)"},{"line_number":6870,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_da73c7e6","line":6867,"range":{"start_line":6866,"start_character":0,"end_line":6867,"end_character":56},"updated":"2019-09-18 17:48:08.000000000","message":"I wonder if this is the problem (see comment on L1822). I can\u0027t say I understand the mechanics of it, but I\u0027m wondering if my passing source_bdms to this new method instead of using the variable as it was done before, we\u0027ve somehow changed it from None to an empty list (or something of that sort), and were not hitting code that we used to hit.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06f27efed8f82693fa92a201d0d659dd83211de","unresolved":false,"context_lines":[{"line_number":6863,"context_line":"        try:"},{"line_number":6864,"context_line":"            if (\u0027block_migration\u0027 in migrate_data and"},{"line_number":6865,"context_line":"                    migrate_data.block_migration):"},{"line_number":6866,"context_line":"                block_device_info \u003d self._get_instance_block_device_info("},{"line_number":6867,"context_line":"                    context, instance, bdms\u003dsource_bdms)"},{"line_number":6868,"context_line":"                disk \u003d self.driver.get_instance_disk_info("},{"line_number":6869,"context_line":"                    instance, block_device_info\u003dblock_device_info)"},{"line_number":6870,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_5deef122","line":6867,"range":{"start_line":6866,"start_character":0,"end_line":6867,"end_character":56},"in_reply_to":"3fa7e38b_3d28d584","updated":"2019-09-18 18:05:05.000000000","message":"Note that migrate_data.bdms should be set by the driver during pre_live_migration here:\n\nhttps://github.com/openstack/nova/blob/c67057dff34a0054977ae3873d33313c0617b308/nova/virt/libvirt/driver.py#L8882","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"9e1d7dc385233f22382df83c38732c0e06cc798d","unresolved":false,"context_lines":[{"line_number":6863,"context_line":"        try:"},{"line_number":6864,"context_line":"            if (\u0027block_migration\u0027 in migrate_data and"},{"line_number":6865,"context_line":"                    migrate_data.block_migration):"},{"line_number":6866,"context_line":"                block_device_info \u003d self._get_instance_block_device_info("},{"line_number":6867,"context_line":"                    context, instance, bdms\u003dsource_bdms)"},{"line_number":6868,"context_line":"                disk \u003d self.driver.get_instance_disk_info("},{"line_number":6869,"context_line":"                    instance, block_device_info\u003dblock_device_info)"},{"line_number":6870,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_3d28d584","line":6867,"range":{"start_line":6866,"start_character":0,"end_line":6867,"end_character":56},"in_reply_to":"3fa7e38b_da73c7e6","updated":"2019-09-18 17:59:52.000000000","message":"How would that be any different from how things work now? Before we had:\n\nsource_bdms \u003d objects.BlockDeviceMappingList.get_by_instance_uuid(\n                context, instance.uuid)\n\nand passed that to _get_instance_block_device_info, right?\n\nNow we still get source_bdms the same way and pass them through to this new method which passes them to _get_instance_block_device_info.\n\nHere is a recent stacktrace from the failure:\n\nSep 18 03:40:00.017771 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]: DEBUG nova.virt.libvirt.driver [-] [instance: f6d51f6e-e817-49ef-940c-10182adbbad3] Migration operation thread notification {{(pid\u003d19633) thread_finished /opt/stack/nova/nova/virt/libvirt/driver.py:8615}}\nSep 18 03:40:00.017983 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]: Traceback (most recent call last):\nSep 18 03:40:00.018193 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py\", line 461, in fire_timers\nSep 18 03:40:00.018601 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     timer()\nSep 18 03:40:00.018745 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/timer.py\", line 59, in __call__\nSep 18 03:40:00.018972 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     cb(*args, **kw)\nSep 18 03:40:00.019115 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/eventlet/event.py\", line 175, in _do_send\nSep 18 03:40:00.019355 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     waiter.switch(result)\nSep 18 03:40:00.019518 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py\", line 221, in main\nSep 18 03:40:00.019794 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     result \u003d function(*args, **kwargs)\nSep 18 03:40:00.019942 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/opt/stack/nova/nova/utils.py\", line 675, in context_wrapper\nSep 18 03:40:00.020362 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     return func(*args, **kwargs)\nSep 18 03:40:00.020509 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 8272, in _live_migration_operation\nSep 18 03:40:00.020698 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     LOG.error(\"Live Migration failure: %s\", e, instance\u003dinstance)\nSep 18 03:40:00.020848 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py\", line 220, in __exit__\nSep 18 03:40:00.021107 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     self.force_reraise()\nSep 18 03:40:00.021244 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py\", line 196, in force_reraise\nSep 18 03:40:00.021388 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     six.reraise(self.type_, self.value, self.tb)\nSep 18 03:40:00.021541 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 8233, in _live_migration_operation\nSep 18 03:40:00.021691 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     get_vif_config\u003dget_vif_config)\nSep 18 03:40:00.021822 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/opt/stack/nova/nova/virt/libvirt/migration.py\", line 87, in get_updated_guest_xml\nSep 18 03:40:00.022257 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     xml_doc \u003d _update_volume_xml(xml_doc, migrate_data, get_volume_config)\nSep 18 03:40:00.022442 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/opt/stack/nova/nova/virt/libvirt/migration.py\", line 185, in _update_volume_xml\nSep 18 03:40:00.022592 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     migrate_bdm_info \u003d migrate_data.bdms\nSep 18 03:40:00.022729 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/oslo_versionedobjects/base.py\", line 67, in getter\nSep 18 03:40:00.023334 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     self.obj_load_attr(name)\nSep 18 03:40:00.023524 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:   File \"/usr/local/lib/python2.7/dist-packages/oslo_versionedobjects/base.py\", line 603, in obj_load_attr\nSep 18 03:40:00.023716 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]:     _(\"Cannot load \u0027%s\u0027 in the base class\") % attrname)\nSep 18 03:40:00.023909 ubuntu-bionic-rax-iad-0011433993 nova-compute[19633]: NotImplementedError: Cannot load \u0027bdms\u0027 in the base class\n\nSo migrate_data.bdms isn\u0027t getting populated during pre_live_migration which should happen on the dest host.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"8fdb7db86d0ea9c1d12b63c94578f91e1c610202","unresolved":false,"context_lines":[{"line_number":6863,"context_line":"        try:"},{"line_number":6864,"context_line":"            if (\u0027block_migration\u0027 in migrate_data and"},{"line_number":6865,"context_line":"                    migrate_data.block_migration):"},{"line_number":6866,"context_line":"                block_device_info \u003d self._get_instance_block_device_info("},{"line_number":6867,"context_line":"                    context, instance, bdms\u003dsource_bdms)"},{"line_number":6868,"context_line":"                disk \u003d self.driver.get_instance_disk_info("},{"line_number":6869,"context_line":"                    instance, block_device_info\u003dblock_device_info)"},{"line_number":6870,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_3d4d3572","line":6867,"range":{"start_line":6866,"start_character":0,"end_line":6867,"end_character":56},"in_reply_to":"3fa7e38b_da73c7e6","updated":"2019-09-18 17:54:54.000000000","message":"I\u0027m thinking of [1]: \"Python’s default arguments are evaluated once when the function is defined, not each time the function is called\" - but there\u0027s not default argument here. Like I said on IRC, more of an intuition than anything else.\n\n[1] https://docs.python-guide.org/writing/gotchas/","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06f27efed8f82693fa92a201d0d659dd83211de","unresolved":false,"context_lines":[{"line_number":6884,"context_line":"                    # TODO(mriedem): The \"block_migration\" parameter passed"},{"line_number":6885,"context_line":"                    # here is not actually used in pre_live_migration but it"},{"line_number":6886,"context_line":"                    # is not optional in the RPC interface either."},{"line_number":6887,"context_line":"                    migrate_data \u003d self.compute_rpcapi.pre_live_migration("},{"line_number":6888,"context_line":"                        context, instance,"},{"line_number":6889,"context_line":"                        block_migration, disk, dest, migrate_data)"},{"line_number":6890,"context_line":"                LOG.info(\u0027Took %0.2f seconds for pre_live_migration on \u0027"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_ddd98183","line":6887,"updated":"2019-09-18 18:05:05.000000000","message":"We call pre_live_migration here (RPC call to the dest and get back the migrate_data that was updated by the dest host virt driver)...AND THAT IS THE PROBLEM. This overwrites the migrate_data from the function signature but that doesn\u0027t update it by reference, so we have to return this and use it to pass down to the driver.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0e2c8835b94f1cae053ddb408b44588351ba383e","unresolved":false,"context_lines":[{"line_number":6941,"context_line":""},{"line_number":6942,"context_line":"    def _do_live_migration(self, context, dest, instance, block_migration,"},{"line_number":6943,"context_line":"                           migration, migrate_data):"},{"line_number":6944,"context_line":"        # NOTE(danms): We should enhance the RT to account for migrations"},{"line_number":6945,"context_line":"        # and use the status field to denote when the accounting has been"},{"line_number":6946,"context_line":"        # done on source/destination. For now, this is just here for status"},{"line_number":6947,"context_line":"        # reporting"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_d12c219b","line":6944,"range":{"start_line":6944,"start_character":23,"end_line":6944,"end_character":73},"updated":"2019-09-17 19:52:30.000000000","message":"This comment is stale since we now have NUMA-aware live migration which performs a move claim.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"9e1d7dc385233f22382df83c38732c0e06cc798d","unresolved":false,"context_lines":[{"line_number":6941,"context_line":""},{"line_number":6942,"context_line":"    def _do_live_migration(self, context, dest, instance, block_migration,"},{"line_number":6943,"context_line":"                           migration, migrate_data):"},{"line_number":6944,"context_line":"        # NOTE(danms): We should enhance the RT to account for migrations"},{"line_number":6945,"context_line":"        # and use the status field to denote when the accounting has been"},{"line_number":6946,"context_line":"        # done on source/destination. For now, this is just here for status"},{"line_number":6947,"context_line":"        # reporting"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_9a2b0f93","line":6944,"range":{"start_line":6944,"start_character":23,"end_line":6944,"end_character":73},"in_reply_to":"3fa7e38b_118e3978","updated":"2019-09-18 17:59:52.000000000","message":"Right, I said NUMA-aware, implying instances with a NUMA topology.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"fda98b0262c9c38e329a5d63127befae06f9743c","unresolved":false,"context_lines":[{"line_number":6941,"context_line":""},{"line_number":6942,"context_line":"    def _do_live_migration(self, context, dest, instance, block_migration,"},{"line_number":6943,"context_line":"                           migration, migrate_data):"},{"line_number":6944,"context_line":"        # NOTE(danms): We should enhance the RT to account for migrations"},{"line_number":6945,"context_line":"        # and use the status field to denote when the accounting has been"},{"line_number":6946,"context_line":"        # done on source/destination. For now, this is just here for status"},{"line_number":6947,"context_line":"        # reporting"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_118e3978","line":6944,"range":{"start_line":6944,"start_character":23,"end_line":6944,"end_character":73},"in_reply_to":"3fa7e38b_d12c219b","updated":"2019-09-17 20:02:19.000000000","message":"Not entirely - only instances with NUMA topologies get a MoveClaim. If we do that allocation-style pinning that we tried but finally left out, all live migrations will get claims, and this comment can be removed.","commit_id":"494cbbfc46775eef713b2d1bdc904e3edf4f9070"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"1068d2cda726f93badab9a883befcb3184e75c1e","unresolved":false,"context_lines":[{"line_number":6843,"context_line":"        :param instance: Instance object being live migrated"},{"line_number":6844,"context_line":"        :param block_migration: If true, prepare for block migration."},{"line_number":6845,"context_line":"        :param migration: Migration object tracking this operation"},{"line_number":6846,"context_line":"        :param migrate_data: MigrateData object for this operation populated"},{"line_number":6847,"context_line":"            by the destination host compute driver as part of the"},{"line_number":6848,"context_line":"            check_can_live_migrate_destination call."},{"line_number":6849,"context_line":"        :param source_bdms: BlockDeviceMappingList of BDMs currently attached"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa7e38b_1d9fce5f","line":6846,"range":{"start_line":6846,"start_character":15,"end_line":6846,"end_character":27},"updated":"2019-09-23 16:38:57.000000000","message":"It kinda looks like this might get partially mutated as a result of this call, rendering it useless to the caller. Which was the case before as well... but since you\u0027re documenting, it could be worth mentioning (or adding a TODO to make the interfaces cleaner).","commit_id":"0ff01646accf61f3654cd175c5e29ea2d80e7b2b"}]}