)]}'
{"specs/stein/approved/live-migration-per-instance-timeout-stein.rst":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"97fcdb529f9e64fe9c4f7d070321c05c8c80f463","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Live-Migration per instance timeout"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/live-migration-per-instance-timeout"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Add a new microversion to live-migrate API to abort or force complete any"},{"line_number":14,"context_line":"libvirt live-migration operation after a given timeout."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_4ac6639c","line":11,"range":{"start_line":11,"start_character":0,"end_line":11,"end_character":79},"updated":"2018-09-07 19:38:53.000000000","message":"This doesn\u0027t match the name of the spec filename.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":35,"context_line":"to change the related config value to better fit and restart compute services"},{"line_number":36,"context_line":"which makes live-migration experience very unpleasant."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Given the recent removal of the progress timeout, we have discussed with"},{"line_number":39,"context_line":"operators that they would like to customize the timeout per live-migration"},{"line_number":40,"context_line":"operation. Based on the VM involved and the cost of not moving the VM, they"},{"line_number":41,"context_line":"can make the call of how long to wait. In a similar way, they want to decide"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_9f6cbb07","line":38,"range":{"start_line":38,"start_character":0,"end_line":38,"end_character":48},"updated":"2018-09-07 19:36:36.000000000","message":"This should be updated, it wasn\u0027t removed, nor was it recent (Ocata).\n\nSo I\u0027d say something like, with the deprecation of the option https://docs.openstack.org/nova/rocky/configuration/config.html#libvirt.live_migration_progress_timeout in Ocata, we have discussed....","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":35,"context_line":"to change the related config value to better fit and restart compute services"},{"line_number":36,"context_line":"which makes live-migration experience very unpleasant."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Given the recent removal of the progress timeout, we have discussed with"},{"line_number":39,"context_line":"operators that they would like to customize the timeout per live-migration"},{"line_number":40,"context_line":"operation. Based on the VM involved and the cost of not moving the VM, they"},{"line_number":41,"context_line":"can make the call of how long to wait. In a similar way, they want to decide"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_ca2ef341","line":38,"range":{"start_line":38,"start_character":0,"end_line":38,"end_character":48},"in_reply_to":"3f79a3b5_9f6cbb07","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":35,"context_line":"to change the related config value to better fit and restart compute services"},{"line_number":36,"context_line":"which makes live-migration experience very unpleasant."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Given the recent removal of the progress timeout, we have discussed with"},{"line_number":39,"context_line":"operators that they would like to customize the timeout per live-migration"},{"line_number":40,"context_line":"operation. Based on the VM involved and the cost of not moving the VM, they"},{"line_number":41,"context_line":"can make the call of how long to wait. In a similar way, they want to decide"},{"line_number":42,"context_line":"if they should abort after that timeout (avoiding the VM having any more"},{"line_number":43,"context_line":"downtime than ``libvirt.live_migration_downtime``), or force the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_bf31770d","line":40,"range":{"start_line":38,"start_character":50,"end_line":40,"end_character":10},"updated":"2018-09-07 19:36:36.000000000","message":"I don\u0027t know where this discussion happened. I was looking through the BM/VM platform etherpad from the boston forum and don\u0027t see it - maybe John would remember.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":35,"context_line":"to change the related config value to better fit and restart compute services"},{"line_number":36,"context_line":"which makes live-migration experience very unpleasant."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Given the recent removal of the progress timeout, we have discussed with"},{"line_number":39,"context_line":"operators that they would like to customize the timeout per live-migration"},{"line_number":40,"context_line":"operation. Based on the VM involved and the cost of not moving the VM, they"},{"line_number":41,"context_line":"can make the call of how long to wait. In a similar way, they want to decide"},{"line_number":42,"context_line":"if they should abort after that timeout (avoiding the VM having any more"},{"line_number":43,"context_line":"downtime than ``libvirt.live_migration_downtime``), or force the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_8ad9db56","line":40,"range":{"start_line":38,"start_character":50,"end_line":40,"end_character":10},"in_reply_to":"3f79a3b5_bf31770d","updated":"2018-09-07 19:53:45.000000000","message":"Found something sort of here:\n\nhttps://etherpad.openstack.org/p/nova-ptg-pike","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":64,"context_line":"Use Cases"},{"line_number":65,"context_line":"---------"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"* Operators want to patch a host and want to move all the VM\u0027s out of that"},{"line_number":68,"context_line":"  host. In this case they want to force a VM to move when timeout is reached"},{"line_number":69,"context_line":"  because they find the risk of possible needing to reboot the VM less"},{"line_number":70,"context_line":"  acceptable than pausing the VM to make it move."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_5f2063b0","line":67,"range":{"start_line":67,"start_character":58,"end_line":67,"end_character":62},"updated":"2018-09-07 19:36:36.000000000","message":"VMs","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":87,"context_line":""},{"line_number":88,"context_line":"* ``on_timeout`` - This optional parameter can be set to"},{"line_number":89,"context_line":"  ``force_complete`` or ``abort``. This will override the config option:"},{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_0a194b78","line":90,"range":{"start_line":90,"start_character":2,"end_line":90,"end_character":75},"updated":"2018-09-07 19:36:36.000000000","message":"We should mention this is added as part of the dependent blueprint live-migration-force-after-timeout.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":87,"context_line":""},{"line_number":88,"context_line":"* ``on_timeout`` - This optional parameter can be set to"},{"line_number":89,"context_line":"  ``force_complete`` or ``abort``. This will override the config option:"},{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_0ac62b69","line":90,"range":{"start_line":90,"start_character":2,"end_line":90,"end_character":75},"in_reply_to":"3f79a3b5_0a194b78","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"},{"line_number":94,"context_line":"least the service version that matches when this feature was added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"To address issue with ramp up time, we propose to spend half of the specified"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_ea796f92","line":93,"range":{"start_line":93,"start_character":8,"end_line":93,"end_character":16},"updated":"2018-09-07 19:36:36.000000000","message":"parameter","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"},{"line_number":94,"context_line":"least the service version that matches when this feature was added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"To address issue with ramp up time, we propose to spend half of the specified"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_2ac1a74e","line":93,"range":{"start_line":93,"start_character":8,"end_line":93,"end_character":16},"in_reply_to":"3f79a3b5_ea796f92","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":89,"context_line":"  ``force_complete`` or ``abort``. This will override the config option:"},{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"},{"line_number":94,"context_line":"least the service version that matches when this feature was added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"To address issue with ramp up time, we propose to spend half of the specified"},{"line_number":97,"context_line":"completion timeout ramping up to maximum downtime as normal. After that, we"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_4a3c23c7","line":94,"range":{"start_line":92,"start_character":0,"end_line":94,"end_character":67},"updated":"2018-09-07 19:36:36.000000000","message":"This is a bit heavy, the timeout applies on the source host, so we could just check in the API if the source compute in the live migration operation is new enough to handle this, rather than the entire deployment. Similar to what we did with https://blueprints.launchpad.net/nova/+spec/abort-live-migration-in-queued-status.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":89,"context_line":"  ``force_complete`` or ``abort``. This will override the config option:"},{"line_number":90,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":93,"context_line":"timeout paramter and before all compute nodes have been upgraded to report at"},{"line_number":94,"context_line":"least the service version that matches when this feature was added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"To address issue with ramp up time, we propose to spend half of the specified"},{"line_number":97,"context_line":"completion timeout ramping up to maximum downtime as normal. After that, we"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_caa113ab","line":94,"range":{"start_line":92,"start_character":0,"end_line":94,"end_character":67},"in_reply_to":"3f79a3b5_4a3c23c7","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":107,"context_line":"VM being live-migrated. However this is far from convenient, and can lead to"},{"line_number":108,"context_line":"races in timeouts happening just before calling ``force-complete``."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"There are many other ways we could modify the downtime ramp up logic. Given"},{"line_number":111,"context_line":"the discussions on re-working that logic we just do the minimum to ensure"},{"line_number":112,"context_line":"``libvirt.live_migration_downtime`` is reached before we hit the timeout"},{"line_number":113,"context_line":"specified by the operator."},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_2a2d278e","line":111,"range":{"start_line":110,"start_character":70,"end_line":111,"end_character":15},"updated":"2018-09-07 19:36:36.000000000","message":"Again I wish this actually referred to something that was recorded.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":107,"context_line":"VM being live-migrated. However this is far from convenient, and can lead to"},{"line_number":108,"context_line":"races in timeouts happening just before calling ``force-complete``."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"There are many other ways we could modify the downtime ramp up logic. Given"},{"line_number":111,"context_line":"the discussions on re-working that logic we just do the minimum to ensure"},{"line_number":112,"context_line":"``libvirt.live_migration_downtime`` is reached before we hit the timeout"},{"line_number":113,"context_line":"specified by the operator."},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_4a7e6319","line":111,"range":{"start_line":110,"start_character":70,"end_line":111,"end_character":15},"in_reply_to":"3f79a3b5_2a2d278e","updated":"2018-09-07 19:53:45.000000000","message":"Maybe it mentions https://review.openstack.org/#/c/430218/ and https://review.openstack.org/#/c/408002/ ?","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":146,"context_line":"    {"},{"line_number":147,"context_line":"        \"timeout_seconds\": {"},{"line_number":148,"context_line":"            \"type\": \"integer\","},{"line_number":149,"context_line":"            \"minimum\": 0"},{"line_number":150,"context_line":"        }"},{"line_number":151,"context_line":"    }"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_0a470b4e","line":149,"range":{"start_line":149,"start_character":23,"end_line":149,"end_character":24},"updated":"2018-09-07 19:36:36.000000000","message":"What does the 0 mean? Disable? I guess so if it\u0027s matching the config option:\n\nhttps://docs.openstack.org/nova/rocky/configuration/config.html#libvirt.live_migration_completion_timeout","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":146,"context_line":"    {"},{"line_number":147,"context_line":"        \"timeout_seconds\": {"},{"line_number":148,"context_line":"            \"type\": \"integer\","},{"line_number":149,"context_line":"            \"minimum\": 0"},{"line_number":150,"context_line":"        }"},{"line_number":151,"context_line":"    }"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_6a9ebf54","line":149,"range":{"start_line":149,"start_character":23,"end_line":149,"end_character":24},"in_reply_to":"3f79a3b5_0a470b4e","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":215,"context_line":"Dependencies"},{"line_number":216,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"We first need the configuration added for the default timeout action:"},{"line_number":219,"context_line":"https://blueprints.launchpad.net/nova/+spec/live-migration-force-after-timeout"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_4a6ae3b2","line":218,"range":{"start_line":218,"start_character":18,"end_line":218,"end_character":31},"updated":"2018-09-07 19:36:36.000000000","message":"Specifically call out the ``libvirt.live_migration_action_on_timeout`` option.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":215,"context_line":"Dependencies"},{"line_number":216,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"We first need the configuration added for the default timeout action:"},{"line_number":219,"context_line":"https://blueprints.launchpad.net/nova/+spec/live-migration-force-after-timeout"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_2aa447a2","line":218,"range":{"start_line":218,"start_character":18,"end_line":218,"end_character":31},"in_reply_to":"3f79a3b5_4a6ae3b2","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":221,"context_line":"Testing"},{"line_number":222,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Need new tempest tests for the new API."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Look into busy workloads inside VMs to test the above API in the gate\u0027s"},{"line_number":227,"context_line":"live-migration job."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_ca9b53c6","line":224,"range":{"start_line":224,"start_character":0,"end_line":224,"end_character":39},"updated":"2018-09-07 19:36:36.000000000","message":"As noted in the other spec, this won\u0027t work with tempest tests, it would race. We need to test this with in-tree functional tests.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":221,"context_line":"Testing"},{"line_number":222,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Need new tempest tests for the new API."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Look into busy workloads inside VMs to test the above API in the gate\u0027s"},{"line_number":227,"context_line":"live-migration job."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_8aaebb7e","line":224,"range":{"start_line":224,"start_character":0,"end_line":224,"end_character":39},"in_reply_to":"3f79a3b5_ca9b53c6","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Need new tempest tests for the new API."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Look into busy workloads inside VMs to test the above API in the gate\u0027s"},{"line_number":227,"context_line":"live-migration job."},{"line_number":228,"context_line":""},{"line_number":229,"context_line":"Documentation Impact"},{"line_number":230,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_aaa4d707","line":227,"range":{"start_line":226,"start_character":0,"end_line":227,"end_character":19},"updated":"2018-09-07 19:36:36.000000000","message":"I still think this would likely be too unpredictable in the gate based on the node provider the job runs on. We could always add it in later, but I wouldn\u0027t block on this.","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Need new tempest tests for the new API."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Look into busy workloads inside VMs to test the above API in the gate\u0027s"},{"line_number":227,"context_line":"live-migration job."},{"line_number":228,"context_line":""},{"line_number":229,"context_line":"Documentation Impact"},{"line_number":230,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_aab137e2","line":227,"range":{"start_line":226,"start_character":0,"end_line":227,"end_character":19},"in_reply_to":"3f79a3b5_aaa4d707","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":231,"context_line":""},{"line_number":232,"context_line":"Need to update api-ref with details of the new API."},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"Should also update the API concept guide to cover how best to use"},{"line_number":235,"context_line":"live-migration with all these new APIs we have added."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"References"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_ca84b360","line":234,"updated":"2018-09-07 19:36:36.000000000","message":"https://docs.openstack.org/nova/latest/admin/configuring-migrations.html#kvm-libvirt","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":231,"context_line":""},{"line_number":232,"context_line":"Need to update api-ref with details of the new API."},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"Should also update the API concept guide to cover how best to use"},{"line_number":235,"context_line":"live-migration with all these new APIs we have added."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"References"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_4acc2369","line":234,"in_reply_to":"3f79a3b5_ca84b360","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a08a83fd2b080e162cf468eaad180d06ad83870d","unresolved":false,"context_lines":[{"line_number":249,"context_line":"   * - Release Name"},{"line_number":250,"context_line":"     - Description"},{"line_number":251,"context_line":"   * - Pike"},{"line_number":252,"context_line":"     - Introduced"},{"line_number":253,"context_line":"   * - Stein"},{"line_number":254,"context_line":"     - Reproposed"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_aa7db76d","line":252,"range":{"start_line":252,"start_character":7,"end_line":252,"end_character":17},"updated":"2018-09-07 19:36:36.000000000","message":"Approved but not implemented","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38e1ecba5e332d92bd1e6fb42900b5f4eae9e54f","unresolved":false,"context_lines":[{"line_number":249,"context_line":"   * - Release Name"},{"line_number":250,"context_line":"     - Description"},{"line_number":251,"context_line":"   * - Pike"},{"line_number":252,"context_line":"     - Introduced"},{"line_number":253,"context_line":"   * - Stein"},{"line_number":254,"context_line":"     - Reproposed"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f79a3b5_2abd27b0","line":252,"range":{"start_line":252,"start_character":7,"end_line":252,"end_character":17},"in_reply_to":"3f79a3b5_aa7db76d","updated":"2018-09-07 19:53:45.000000000","message":"Done","commit_id":"b7365bf56fc3261778c7d8829a5976b36193e1c9"}],"specs/stein/approved/live-migration-per-instance-timeout.rst":[{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"785485dbb45f93ec2e38d54b57133e88abb6e0d0","unresolved":false,"context_lines":[{"line_number":32,"context_line":"With current live-migrate API and config options, operators do not have"},{"line_number":33,"context_line":"fine-grained control over per instance live-migrate operations. If they want to"},{"line_number":34,"context_line":"treat any particular instance live-migrate operation different then they have"},{"line_number":35,"context_line":"to change the related config value to better fit and restart compute services"},{"line_number":36,"context_line":"which makes live-migration experience very unpleasant."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Given the `deprecation`_ of the ``libvirt.live_migration_progress_timeout``"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_dfa21e9a","line":35,"range":{"start_line":35,"start_character":3,"end_line":35,"end_character":77},"updated":"2018-09-09 18:48:03.000000000","message":"Presumably we could make these hot reloadable options so that the restart is not necessary. But, obviously if the migrations aren\u0027t proceeding it\u0027s a bit of a race to tweak this value in the config to avoid wasting the work done already.","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4ed0ed7f3aab2ce21276808627400406c6fac03f","unresolved":false,"context_lines":[{"line_number":94,"context_line":"  ``libvirt.live_migration_action_on_timeout``, that defaults to ``abort``."},{"line_number":95,"context_line":"  Note that the ``libvirt.live_migration_action_on_timeout`` is introduced in"},{"line_number":96,"context_line":"  dependent `blueprint live-migration-force-after-timeout`_."},{"line_number":97,"context_line":""},{"line_number":98,"context_line":".. _blueprint live-migration-force-after-timeout: https://blueprints.launchpad.net/nova/+spec/live-migration-force-after-timeout"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_f1f4dfd2","line":97,"updated":"2018-09-10 16:53:44.000000000","message":"While we\u0027re adding new parameters, I have a few additional suggestions for other parameters we might want to consider adding to the migration request.\n\n1) \u0027max_downtime\u0027 -- rather than allowing unlimited downtime by forcing the migration, we could specify an increased (but still limited) downtime for this migration.\n\n2) \u0027permit_auto_converge\u0027 -- specify whether or not auto-converge is requested/blocked for this migration.  Some applications might be highly sensitive to reduced performance and we\u0027d be better off forcing the workload elsewhere.\n\n3) \u0027permit_post_copy\u0027 -- specify for this migration whether forced migration is done via post-copy or pause.\n\nI\u0027m not sure if these all translate well for hypervisors other than libvirt, but it would be nice to have finer-grained control than host-level configs.","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"785485dbb45f93ec2e38d54b57133e88abb6e0d0","unresolved":false,"context_lines":[{"line_number":95,"context_line":"  Note that the ``libvirt.live_migration_action_on_timeout`` is introduced in"},{"line_number":96,"context_line":"  dependent `blueprint live-migration-force-after-timeout`_."},{"line_number":97,"context_line":""},{"line_number":98,"context_line":".. _blueprint live-migration-force-after-timeout: https://blueprints.launchpad.net/nova/+spec/live-migration-force-after-timeout"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":101,"context_line":"timeout parameter and before the source compute service have been upgraded to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_ffcb5a46","line":98,"updated":"2018-09-09 18:48:03.000000000","message":"I guess I\u0027m not sure why we\u0027re doing the config option for the action if we\u0027re also going to allow it via the API as well.","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"a3e6bd00639454b0765ff76c37809f2639af5059","unresolved":false,"context_lines":[{"line_number":95,"context_line":"  Note that the ``libvirt.live_migration_action_on_timeout`` is introduced in"},{"line_number":96,"context_line":"  dependent `blueprint live-migration-force-after-timeout`_."},{"line_number":97,"context_line":""},{"line_number":98,"context_line":".. _blueprint live-migration-force-after-timeout: https://blueprints.launchpad.net/nova/+spec/live-migration-force-after-timeout"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"To help upgrades, we return 400 for any requests containing either of the new"},{"line_number":101,"context_line":"timeout parameter and before the source compute service have been upgraded to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_d7099e32","line":98,"in_reply_to":"3f79a3b5_ffcb5a46","updated":"2018-09-10 09:01:24.000000000","message":"Yes, it seems if we have this on_timeout params we can drop live_migration_action_on_timeout configuration directly.\n\nHow about we first remove/deprecated the config which added in bp/live-migration-force-after-timeout, and then add this on_timeout parameter?","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4ed0ed7f3aab2ce21276808627400406c6fac03f","unresolved":false,"context_lines":[{"line_number":101,"context_line":"timeout parameter and before the source compute service have been upgraded to"},{"line_number":102,"context_line":"report at least the service version that matches when this feature was added."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"To address issue with ramp up time, we propose to spend half of the specified"},{"line_number":105,"context_line":"completion timeout ramping up to maximum downtime as normal. After that, we"},{"line_number":106,"context_line":"jump up to ``libvirt.live_migration_downtime``. This will ensure VM will spend"},{"line_number":107,"context_line":"half of the specified timeout with the best chance of letting live-migration"},{"line_number":108,"context_line":"complete without having to abort or force-complete."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"Alternatives"},{"line_number":111,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_1efdea3a","line":108,"range":{"start_line":104,"start_character":0,"end_line":108,"end_character":51},"updated":"2018-09-10 16:53:44.000000000","message":"Presumably this means we would now ignore (and deprecate/remove) the config option \u0027libvirt.live_migration_downtime_delay\u0027 since it\u0027ll be automatically calculated from the overall timeout?   (I\u0027m in favor of this, was going to propose something similar.)","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"785485dbb45f93ec2e38d54b57133e88abb6e0d0","unresolved":false,"context_lines":[{"line_number":119,"context_line":"the `discussions`_ on re-working that logic we just do the minimum to ensure"},{"line_number":120,"context_line":"``libvirt.live_migration_downtime`` is reached before we hit the timeout"},{"line_number":121,"context_line":"specified by the operator."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":".. _discussions: https://etherpad.openstack.org/p/nova-ptg-pike"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_bfeac29c","line":122,"updated":"2018-09-09 18:48:03.000000000","message":"Another option here is to just build this whole thing into the client, right? It seems like it would be trivial to just make something like this:\n\n openstack server live-migrate --timeout\u003d300 --action\u003dforce\n\nwhich would just poll the migration, and if it hasn\u0027t completed by the deadline, either force or abort, depending on the flags. That would make the server-side configs be the upper limit, and that logic would all remain the same as it is today.\n\nObviously that may not be as desirable for migrations that are expected to run very long term as the client has to remain running, but it seems a little more straightforward to me.\n\nI guess it just seems like it\u0027s hard to reason about what the behavior will be, if we take a timeout and action via the api, we have a timeout and action in the config, and we have some complicated server-side logic that impacts all of it.\n\nAlso, the client-side behavior avoids the nasty concern over this being a libvirt-only thing. What happens if someone specifies a timeout and action in the api, but is migrating something on xen or hyperv? I dunno what those drivers have in the way of these same settings, but hopefully they either support abort/force-complete or return something useful to the api client trying to do those actions.","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"34a0b8a2bfd77c3f1e9dc5ed39b0889730ecd9f3","unresolved":false,"context_lines":[{"line_number":119,"context_line":"the `discussions`_ on re-working that logic we just do the minimum to ensure"},{"line_number":120,"context_line":"``libvirt.live_migration_downtime`` is reached before we hit the timeout"},{"line_number":121,"context_line":"specified by the operator."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":".. _discussions: https://etherpad.openstack.org/p/nova-ptg-pike"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_fa98b67c","line":122,"in_reply_to":"3f79a3b5_11387b39","updated":"2018-12-18 22:47:42.000000000","message":"\u003e I dunno what those drivers have in the way of these same settings, but hopefully they either support abort/force-complete or return something useful to the api client trying to do those actions.\n\nThe API RPC casts to nova-compute which will call the driver to abort or force-complete which will raise NotImplementedError which will get recorded as a fault on the instance but otherwise not affect the migration. Operators could disable those APIs via policy if they only have computes which don\u0027t support these operations. At the Stein PTG John thought xenapi should be able to support abort/force-complete, but no one is working on it.\n\nI\u0027m not crazy about the client-side alternative since it blocks up the client waiting for that operation to complete. I would think operators doing a lot of live migrations (especially in a public cloud which is where the requirement is coming from for Huawei) would want to fire off the live migration with a given timeout value and action and move on until it\u0027s done (either success or failure), and leave the config options as catch-alls for all other servers they don\u0027t necessarily need to care as much about.","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4ed0ed7f3aab2ce21276808627400406c6fac03f","unresolved":false,"context_lines":[{"line_number":119,"context_line":"the `discussions`_ on re-working that logic we just do the minimum to ensure"},{"line_number":120,"context_line":"``libvirt.live_migration_downtime`` is reached before we hit the timeout"},{"line_number":121,"context_line":"specified by the operator."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":".. _discussions: https://etherpad.openstack.org/p/nova-ptg-pike"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_11387b39","line":122,"in_reply_to":"3f79a3b5_bfeac29c","updated":"2018-09-10 16:53:44.000000000","message":"I\u0027m sympathetic to this idea, but it\u0027s necessarily lacking in nuance...the instance hits the deadline and wham!\n\nOne big thing that\u0027s missing if we do it client-side is the ability to recalculate the downtime steps to ramp up faster...otherwise for a shorter overall timeout we might time out while still using a relatively low max downtime.\n\nSimilarly it would rule out supporting any of the other additional options I proposed.  Which might be considered a good thing as far as code complexity. :)","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e082f53e013fa3673dae180aef7b32e9e7feb1b","unresolved":false,"context_lines":[{"line_number":250,"context_line":"Need to update api-ref with details of the new API."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"Should also update the `API guide`_ to cover how best to use"},{"line_number":253,"context_line":"live-migration with all these new APIs we have added."},{"line_number":254,"context_line":""},{"line_number":255,"context_line":".. _API guide: https://docs.openstack.org/nova/latest/admin/configuring-migrations.html#kvm-libvirt"},{"line_number":256,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f79a3b5_8a58fb89","line":253,"updated":"2018-09-07 20:02:53.000000000","message":"I guess this too:\n\nhttps://docs.openstack.org/nova/latest/admin/live-migration-usage.html","commit_id":"df2baec3e26a8f288abf53c69f9e0036cc134055"}]}