)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"70202862225e45470fa244643ce18134be964227","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"This spec adds support to allow abort live migrations in ``queued``"},{"line_number":24,"context_line":"status."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Change-Id: I61881e2dc079412631c59a70567dd4e09eb257ca"},{"line_number":27,"context_line":"Implement: blueprint abort-live-migration-in-queued-status"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"bf659307_283b659c","line":25,"updated":"2018-03-28 13:12:52.000000000","message":"nit: Need the APIImpact tag on this.","commit_id":"3f5a1dfbb282828931d23f775b0ff680dd6b5f58"}],"specs/rocky/approved/abort-live-migration-in-queue.rst":[{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"4e834bde7bbf724bc5e5d631ae758cb4ecbb3ad6","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Problem description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"The functionality of abort live migration was added in microversion 2.24,"},{"line_number":20,"context_line":"and currently only migrations in ``running`` status are allowed to be"},{"line_number":21,"context_line":"aborted."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_cd6199ab","line":19,"range":{"start_line":19,"start_character":42,"end_line":19,"end_character":72},"updated":"2018-01-30 01:05:35.000000000","message":"For ease of review (or just for reference), a link to that spec (I\u0027m guessing https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/abort-live-migration.html?) would be nice.","commit_id":"4552d969c8a9003728f786324c035bc07ffad218"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"45cd8f33410755ce498424fb5330853c41394faf","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Problem description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"The functionality of abort live migration was added in microversion 2.24,"},{"line_number":20,"context_line":"and currently only migrations in ``running`` status are allowed to be"},{"line_number":21,"context_line":"aborted."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"df7087c5_bf7d3bb3","line":19,"range":{"start_line":19,"start_character":42,"end_line":19,"end_character":72},"in_reply_to":"5f93b717_cd6199ab","updated":"2018-03-14 01:49:20.000000000","message":"Done","commit_id":"4552d969c8a9003728f786324c035bc07ffad218"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"c9317caf15db0bb43b9989831202dff60c4830ba","unresolved":false,"context_lines":[{"line_number":41,"context_line":"Proposed change"},{"line_number":42,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Add an API microversion to ``DELETE /servers/{id}/migrations/{migration_id}``"},{"line_number":45,"context_line":"API to allow abort live migration in ``queued`` status."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Alternatives"},{"line_number":48,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1f9dbf25_df83c003","line":45,"range":{"start_line":44,"start_character":0,"end_line":45,"end_character":55},"updated":"2018-03-02 12:21:37.000000000","message":"The code that serializes the live migrations on the compute node uses a python semaphore.  We\u0027ll need toeither use a different serialization mechanism that would allow us to wake up those threads (to allow them to clean themselves up), or kill them, or ensure that they\u0027ll suicide when they wake up.","commit_id":"4552d969c8a9003728f786324c035bc07ffad218"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"45cd8f33410755ce498424fb5330853c41394faf","unresolved":false,"context_lines":[{"line_number":41,"context_line":"Proposed change"},{"line_number":42,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Add an API microversion to ``DELETE /servers/{id}/migrations/{migration_id}``"},{"line_number":45,"context_line":"API to allow abort live migration in ``queued`` status."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Alternatives"},{"line_number":48,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"df7087c5_7f4d6398","line":45,"range":{"start_line":44,"start_character":0,"end_line":45,"end_character":55},"in_reply_to":"1f9dbf25_df83c003","updated":"2018-03-14 01:49:20.000000000","message":"Yeah, I will add more details here.","commit_id":"4552d969c8a9003728f786324c035bc07ffad218"}],"specs/rocky/approved/abort-live-migration-in-queued-status.rst":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0754e62ddccb8d3bd0d94adf4e62882b966cf365","unresolved":false,"context_lines":[{"line_number":20,"context_line":"and currently only migrations in ``running`` status are allowed to be"},{"line_number":21,"context_line":"aborted."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"While the number of live migration requests could be greater than the max"},{"line_number":24,"context_line":"concurrent live migration configuration ``max_concurrent_live_migrations``,"},{"line_number":25,"context_line":"and there will be migrations wait in queue. The migrations could remain in"},{"line_number":26,"context_line":"``queued`` status for a very long time depend on the queue length and the"},{"line_number":27,"context_line":"processing speed."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Admins may want to abort migrations in queue due to time consumption"},{"line_number":30,"context_line":"considerations etc. It will be unreasonable to make admins wait until"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_b0575c26","line":27,"range":{"start_line":23,"start_character":0,"end_line":27,"end_character":17},"updated":"2018-03-16 14:20:37.000000000","message":"max_concurrent_live_migrations defaults to 1, so if an admin is trying to live migrate everything off a node before taking it down for maintenance, I could see the use case here, especially if they are in a hurry.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0754e62ddccb8d3bd0d94adf4e62882b966cf365","unresolved":false,"context_lines":[{"line_number":51,"context_line":"to be ``CONF.max_concurrent_live_migrations``, each incoming migration will"},{"line_number":52,"context_line":"try to acquire this semaphore, if the acquire succeed, the value of the"},{"line_number":53,"context_line":"semaphore will decrease by one, and the status of the migration will turn"},{"line_number":54,"context_line":"to status other than ``queued``, when the value decreased to 0, new incoming"},{"line_number":55,"context_line":"migrations will be blocked(migration status will be ``queued``) until some of"},{"line_number":56,"context_line":"the previous migration was finished(succeed, failed or aborted) and releases"},{"line_number":57,"context_line":"the semaphore."}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_e43d15c4","line":54,"range":{"start_line":54,"start_character":10,"end_line":54,"end_character":31},"updated":"2018-03-16 14:20:37.000000000","message":"Yeah, next status is \u0027preparing\u0027.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0754e62ddccb8d3bd0d94adf4e62882b966cf365","unresolved":false,"context_lines":[{"line_number":54,"context_line":"to status other than ``queued``, when the value decreased to 0, new incoming"},{"line_number":55,"context_line":"migrations will be blocked(migration status will be ``queued``) until some of"},{"line_number":56,"context_line":"the previous migration was finished(succeed, failed or aborted) and releases"},{"line_number":57,"context_line":"the semaphore."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"According to the above mentioned implementation, it is unable to abort a"},{"line_number":60,"context_line":"migration in ``queued`` status as there is actually no QUEUE, so we are"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_e442f53d","line":57,"updated":"2018-03-16 14:20:37.000000000","message":"Also, reading up on http://eventlet.net/doc/modules/semaphore.html, it looks like it\u0027s not a queue at all:\n\n\"The implementation may pick one at random, so the order in which blocked threads are awakened should not be relied on.\"","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"38b7ebbc34ad6108b84b62ca28d2256bf368a4c6","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_faa6450f","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"updated":"2018-03-14 23:38:34.000000000","message":"Are we okay with making this depend on the internal implementation for Semaphore?","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"bcd0d14246e52b6b1b6e3bdb5c51ef2147c07ce1","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_e1aac86f","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_18a98a8e","updated":"2018-03-19 06:59:41.000000000","message":"Hi, Thanks all for the review and suggestions, After reading docs and examples about concurrent.futures.ThreadPoolExecutor, it seems could work, but I have one question about implementation details, As we now uses utils.spawn_n() to generate the thread, and seems the different between ThreadPoolExecutor and today\u0027s semaphore is that seems ThreadPoolExcutor have the ability to create the thread and semaphore is just a blocker, do I have to modify the whole method to use ThreadPoolExcutor instead of spwn_n()?","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"6d52766173970d3280d8ef9f685a3b6e002eac04","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_18a98a8e","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_44b96176","updated":"2018-03-16 16:26:54.000000000","message":"Another option would be to explicitly create a queue of waiters, and use a Condition object per waiting thread.  That would allow us to cancel a single waiting thread and then notify() it of the change in state.\n\nUnder normal circumstances, the code that releases the live migration semaphore would notify() the next waiter in the queue (if any).\n\nUsing futures might be cleaner. :)","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"10f5e07a4ecf3993c0f2b151f4f593365896f8f9","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_c6c45e95","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_dee0b5fd","updated":"2018-03-20 01:11:25.000000000","message":"Thanks a lot for the reply, just want to thought it through :), will update the spec today.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"4623094b2ec1c248a567790bf21b9baf433015e4","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_dee0b5fd","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_e1aac86f","updated":"2018-03-19 15:28:01.000000000","message":"I think the ThreadPoolExecutor replaces the usage of spawn_n, but not totally sure. At this point it\u0027s an implementation detail and you\u0027ll likely need to sort it out during implementation, testing and code review.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"438f2364a58d257c4aab1fb4dbd79c627e4a8a07","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_44b96176","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_e4b0b545","updated":"2018-03-16 14:38:44.000000000","message":"Reading up on: http://pythonhosted.org/futures/\n\nI think the flow is to basically replace the Semaphore we use today with a ThreadPoolExecutor. We submit _do_live_migration calls to that pool, which returns a Future object. We map the Future to a dict, keyed by migration id (uuid?), and then we can later use that to lookup the Future (if it hasn\u0027t already started running) and call Future.cancel() on it.\n\nWhen the compute manager is shutting down, we\u0027ll have to remember to call ThreadPoolExecutor.shutdown() so we can close the pool and cancel any waiting tasks.\n\nI think we can cleanup the dict entries by simply removing the entry once we start running a task and change the migration status to \u0027preparing\u0027.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0754e62ddccb8d3bd0d94adf4e62882b966cf365","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":67,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"},{"line_number":68,"context_line":"  today. We will a new method ``cancel`` to the new class that allow caller"},{"line_number":69,"context_line":"  to delete one of the thread in ``Semaphore._waiters``(the queue inside"},{"line_number":70,"context_line":"  Semaphore). The input of the ``cancel`` method will be a ``thread`` object::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   class CancelableSemaphore(eventlet.semaphore.Semaphore):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_e4b0b545","line":69,"range":{"start_line":68,"start_character":9,"end_line":69,"end_character":55},"in_reply_to":"df7087c5_faa6450f","updated":"2018-03-16 14:20:37.000000000","message":"Yeah I\u0027m not crazy about this idea. Isn\u0027t there anything in the python threading module that can be used here for an actual thread-blocking queue manager?\n\nI\u0027m reading up on Queues and Threads in the standard library, and some articles:\n\nhttps://www.troyfawkes.com/learn-python-multithreading-queues-basics/\n\nhttps://pymotw.com/2/Queue/\n\nSo we can use threads and queues together, but I\u0027m not seeing a simple way of indexing items in the queue so that we can remove them. So rather than a list, we really want a sorted dict, keyed by migration id mapped to the actual thread, with the sort order being when it was added to the dict (queue).\n\nI have to believe there is something like this in python, since I\u0027m pretty sure I\u0027ve done stuff like this before in Java, but that was years ago. But this is really what I\u0027m used to from Java and looks perfect, except it is apparently only available in py3:\n\nhttps://docs.python.org/3/library/concurrent.futures.html\n\nMaybe there is a library for it in py2?\n\nO-ho there is! https://pypi.python.org/pypi/futures\n\nSo I\u0027d suggest we try using that instead. You create a thread pool executor, throw tasks into the pool and then you can cancel them if they haven\u0027t started yet.","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"38b7ebbc34ad6108b84b62ca28d2256bf368a4c6","unresolved":false,"context_lines":[{"line_number":77,"context_line":"        self._waiters.remove(thread)"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"* Initialize ``ComputeManager._live_migration_semaphore`` with this new"},{"line_number":80,"context_line":"  semaphore calss."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Modify ``spawn_n`` method in ``nova\\utils.py`` to return the created thread"},{"line_number":83,"context_line":"  object for further usage."}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_9ab1494a","line":80,"range":{"start_line":80,"start_character":12,"end_line":80,"end_character":17},"updated":"2018-03-14 23:38:34.000000000","message":"class","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"036bd92890f40d58461caa0cf1bc88c13d07e8f9","unresolved":false,"context_lines":[{"line_number":77,"context_line":"        self._waiters.remove(thread)"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"* Initialize ``ComputeManager._live_migration_semaphore`` with this new"},{"line_number":80,"context_line":"  semaphore calss."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Modify ``spawn_n`` method in ``nova\\utils.py`` to return the created thread"},{"line_number":83,"context_line":"  object for further usage."}],"source_content_type":"text/x-rst","patch_set":4,"id":"df7087c5_e0a3f4fe","line":80,"range":{"start_line":80,"start_character":12,"end_line":80,"end_character":17},"in_reply_to":"df7087c5_9ab1494a","updated":"2018-03-15 01:20:57.000000000","message":"Done","commit_id":"143e91a3ec733b9d969fdb66e478df6815c06d9d"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":52,"context_line":"to be ``CONF.max_concurrent_live_migrations``, each incoming migration will"},{"line_number":53,"context_line":"try to acquire this semaphore, if the acquire succeed, the value of the"},{"line_number":54,"context_line":"semaphore will decrease by one, and the status of the migration will turn"},{"line_number":55,"context_line":"to status other than ``queued``, when the value decreased to 0, new incoming"},{"line_number":56,"context_line":"migrations will be blocked(migration status will be ``queued``) until some of"},{"line_number":57,"context_line":"the previous migration was finished(succeed, failed or aborted) and releases"},{"line_number":58,"context_line":"the semaphore."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_65bc38bb","line":55,"range":{"start_line":55,"start_character":31,"end_line":55,"end_character":37},"updated":"2018-03-28 00:18:02.000000000","message":". When the...","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":52,"context_line":"to be ``CONF.max_concurrent_live_migrations``, each incoming migration will"},{"line_number":53,"context_line":"try to acquire this semaphore, if the acquire succeed, the value of the"},{"line_number":54,"context_line":"semaphore will decrease by one, and the status of the migration will turn"},{"line_number":55,"context_line":"to status other than ``queued``, when the value decreased to 0, new incoming"},{"line_number":56,"context_line":"migrations will be blocked(migration status will be ``queued``) until some of"},{"line_number":57,"context_line":"the previous migration was finished(succeed, failed or aborted) and releases"},{"line_number":58,"context_line":"the semaphore."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_ab18e174","line":55,"range":{"start_line":55,"start_character":31,"end_line":55,"end_character":37},"in_reply_to":"bf659307_65bc38bb","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":68,"context_line":"  The size of the Thread Pool will be limited by"},{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that latter. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_a5c1f030","line":71,"range":{"start_line":71,"start_character":52,"end_line":71,"end_character":58},"updated":"2018-03-28 00:18:02.000000000","message":"later","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":68,"context_line":"  The size of the Thread Pool will be limited by"},{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that latter. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_4b290588","line":71,"range":{"start_line":71,"start_character":52,"end_line":71,"end_character":58},"in_reply_to":"bf659307_a5c1f030","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* Add a new ``_waiting_live_migration`` variable to the ``ComputeManager``"},{"line_number":76,"context_line":"  class of the compute node, this will be a dict, and will be initialized"},{"line_number":77,"context_line":"  as an empty dict. We will 1. record the connection between"},{"line_number":78,"context_line":"  ``migration_uuid`` and the ``Future`` object when the thread is created"},{"line_number":79,"context_line":"  in previous step, we will use ``migration_uuid`` as key and ``Future``"},{"line_number":80,"context_line":"  object as value in our dict. 2.remove the corresponding key/value the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_650af88a","line":77,"range":{"start_line":77,"start_character":20,"end_line":77,"end_character":30},"updated":"2018-03-28 00:18:02.000000000","message":"Turn this into a numbered list in this section for readability.","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* Add a new ``_waiting_live_migration`` variable to the ``ComputeManager``"},{"line_number":76,"context_line":"  class of the compute node, this will be a dict, and will be initialized"},{"line_number":77,"context_line":"  as an empty dict. We will 1. record the connection between"},{"line_number":78,"context_line":"  ``migration_uuid`` and the ``Future`` object when the thread is created"},{"line_number":79,"context_line":"  in previous step, we will use ``migration_uuid`` as key and ``Future``"},{"line_number":80,"context_line":"  object as value in our dict. 2.remove the corresponding key/value the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_0b230daa","line":77,"range":{"start_line":77,"start_character":20,"end_line":77,"end_character":30},"in_reply_to":"bf659307_650af88a","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"* Add an API microversion to"},{"line_number":109,"context_line":"  ``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort"},{"line_number":110,"context_line":"  live migration in ``queued`` status."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":113,"context_line":"  shutting down. This can simply done by calling"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_25396040","line":110,"updated":"2018-03-28 00:18:02.000000000","message":"That would be the check here right?\n\nhttps://github.com/openstack/nova/blob/8c7e9cce7e9c37289d76e0ebdfd0f08ef3c1866c/nova/compute/api.py#L4305\n\nSo we\u0027ll have to pass some kind of \u0027allow_queued\u0027 flag or something based on the microversion.","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":110,"context_line":"  live migration in ``queued`` status."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":113,"context_line":"  shutting down. This can simply done by calling"},{"line_number":114,"context_line":"  ``ThreadPoolExecutor.shutdown()``."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_c53f4423","line":113,"range":{"start_line":113,"start_character":33,"end_line":113,"end_character":37},"updated":"2018-03-28 00:18:02.000000000","message":"be done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":111,"context_line":""},{"line_number":112,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":113,"context_line":"  shutting down. This can simply done by calling"},{"line_number":114,"context_line":"  ``ThreadPoolExecutor.shutdown()``."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Alternatives"},{"line_number":117,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_c528a4e3","line":114,"updated":"2018-03-28 00:18:02.000000000","message":"By default this blocks until everything in the pool is executed, which I don\u0027t think we want if we\u0027re trying to stop or restart nova-compute, so this should probably be wait\u003dFalse.\n\nhttps://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor.shutdown\n\nIt also looks like even with wait\u003dFalse it will hold up the process until all currently running threads are done executing. I\u0027m not sure if there are ways around that.","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":111,"context_line":""},{"line_number":112,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":113,"context_line":"  shutting down. This can simply done by calling"},{"line_number":114,"context_line":"  ``ThreadPoolExecutor.shutdown()``."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Alternatives"},{"line_number":117,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_ebd81980","line":114,"in_reply_to":"bf659307_c528a4e3","updated":"2018-03-28 01:13:14.000000000","message":"Hmmm thats true...","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":116,"context_line":"Alternatives"},{"line_number":117,"context_line":"------------"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"We can also do the below mentioned modification in step 1:"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":122,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_90b9882f","line":119,"updated":"2018-03-28 00:18:02.000000000","message":"I don\u0027t think we should even really mention this, since I think extending eventlet and messing with it\u0027s internals is a bad idea.","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":116,"context_line":"Alternatives"},{"line_number":117,"context_line":"------------"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"We can also do the below mentioned modification in step 1:"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"* Add a new ``CancelableSemephore`` class to ``compute\\utils.py``, the class"},{"line_number":122,"context_line":"  will be a subclass of the original ``eventlet.semaphore.Semaphore`` used"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_6b9f29b4","line":119,"in_reply_to":"bf659307_90b9882f","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":148,"context_line":"The proposal would add API microversion to"},{"line_number":149,"context_line":"``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort live"},{"line_number":150,"context_line":"migration in ``queued`` status. When request with API microversion larger"},{"line_number":151,"context_line":"than the newly added microversion, The response will change from"},{"line_number":152,"context_line":"``HTTP 409 Conflict`` to ``HTTP 202 Accepted`` if the status of requested"},{"line_number":153,"context_line":"live migration is in ``queued`` status."},{"line_number":154,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_b0b4cc36","line":151,"range":{"start_line":151,"start_character":35,"end_line":151,"end_character":38},"updated":"2018-03-28 00:18:02.000000000","message":"the","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":148,"context_line":"The proposal would add API microversion to"},{"line_number":149,"context_line":"``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort live"},{"line_number":150,"context_line":"migration in ``queued`` status. When request with API microversion larger"},{"line_number":151,"context_line":"than the newly added microversion, The response will change from"},{"line_number":152,"context_line":"``HTTP 409 Conflict`` to ``HTTP 202 Accepted`` if the status of requested"},{"line_number":153,"context_line":"live migration is in ``queued`` status."},{"line_number":154,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_6b888970","line":151,"range":{"start_line":151,"start_character":35,"end_line":151,"end_character":38},"in_reply_to":"bf659307_b0b4cc36","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ab0a3dea536ff5a9b3f72ab3acacd1546f9facd1","unresolved":false,"context_lines":[{"line_number":149,"context_line":"``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort live"},{"line_number":150,"context_line":"migration in ``queued`` status. When request with API microversion larger"},{"line_number":151,"context_line":"than the newly added microversion, The response will change from"},{"line_number":152,"context_line":"``HTTP 409 Conflict`` to ``HTTP 202 Accepted`` if the status of requested"},{"line_number":153,"context_line":"live migration is in ``queued`` status."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_70887470","line":152,"range":{"start_line":152,"start_character":7,"end_line":152,"end_character":10},"updated":"2018-03-28 00:18:02.000000000","message":"It\u0027s a 400 today:\n\nhttps://github.com/openstack/nova/blob/8c7e9cce7e9c37289d76e0ebdfd0f08ef3c1866c/nova/api/openstack/compute/server_migrations.py#L158","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"b3b4c53847ea3b5f75d9be424a4e78677d048f3e","unresolved":false,"context_lines":[{"line_number":149,"context_line":"``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort live"},{"line_number":150,"context_line":"migration in ``queued`` status. When request with API microversion larger"},{"line_number":151,"context_line":"than the newly added microversion, The response will change from"},{"line_number":152,"context_line":"``HTTP 409 Conflict`` to ``HTTP 202 Accepted`` if the status of requested"},{"line_number":153,"context_line":"live migration is in ``queued`` status."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_4b858559","line":152,"range":{"start_line":152,"start_character":7,"end_line":152,"end_character":10},"in_reply_to":"bf659307_70887470","updated":"2018-03-28 01:13:14.000000000","message":"Done","commit_id":"be349f7e31e4cc0d9096adf0ec15c009f1fba847"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"70202862225e45470fa244643ce18134be964227","unresolved":false,"context_lines":[{"line_number":82,"context_line":""},{"line_number":83,"context_line":"    2.remove the corresponding key/value the first thing if the thread"},{"line_number":84,"context_line":"    successfully acquired the executor and enter ``_do_live_migration()``[2]."},{"line_number":85,"context_line":"    In this way, we will have a queue-like thing to and make them possible to"},{"line_number":86,"context_line":"    get by ``migration_uuid``."},{"line_number":87,"context_line":""},{"line_number":88,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_0861c1a9","line":85,"range":{"start_line":85,"start_character":49,"end_line":85,"end_character":55},"updated":"2018-03-28 13:12:52.000000000","message":"?\n\nto store the queued thread?\n\nor, \"In this way, we will have a queue-like thing and make it possible to get by ``migration_uuid``.\"","commit_id":"3f5a1dfbb282828931d23f775b0ff680dd6b5f58"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"70202862225e45470fa244643ce18134be964227","unresolved":false,"context_lines":[{"line_number":118,"context_line":"Alternatives"},{"line_number":119,"context_line":"------------"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Keep the current implementation."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"Data model impact"},{"line_number":124,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_c85ae9f1","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":32},"updated":"2018-03-28 13:12:52.000000000","message":"Just put None here.","commit_id":"3f5a1dfbb282828931d23f775b0ff680dd6b5f58"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"70202862225e45470fa244643ce18134be964227","unresolved":false,"context_lines":[{"line_number":132,"context_line":"``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort live"},{"line_number":133,"context_line":"migration in ``queued`` status. When request with API microversion larger"},{"line_number":134,"context_line":"than the newly added microversion, the response will change from"},{"line_number":135,"context_line":"``HTTP 400 Conflict`` to ``HTTP 202 Accepted`` if the status of requested"},{"line_number":136,"context_line":"live migration is in ``queued`` status."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_e85d6de9","line":135,"range":{"start_line":135,"start_character":11,"end_line":135,"end_character":19},"updated":"2018-03-28 13:12:52.000000000","message":"BadRequest","commit_id":"3f5a1dfbb282828931d23f775b0ff680dd6b5f58"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"25a951c91b7cdcfa54336758d7d0fc1764cfcee2","unresolved":false,"context_lines":[{"line_number":108,"context_line":"  provided thread is no longer blocked, so we can switch to the workflow of"},{"line_number":109,"context_line":"  abort migration in ``running`` status as is today."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"* Add an API microversion to"},{"line_number":112,"context_line":"  ``DELETE /servers/{id}/migrations/{migration_id}`` API to allow abort"},{"line_number":113,"context_line":"  live migration in ``queued`` status."},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_a23ea288","line":111,"updated":"2018-03-28 22:20:04.000000000","message":"I thought about this later, and it\u0027s an implementation detail, but instances running on older computes (queens) won\u0027t have the cancel code, so if the API tries to RPC cast to those it won\u0027t do what you want, and might break things. We can handle this in the API in a couple of ways though which we can sort out in code review:\n\n1. The API could check the instance.host\u0027s nova-compute service version and make sure it\u0027s new enough for the new support.\n\n2. Pass a flag from the compute API to the compute RPC API client \"live_migration_abort\" method which can check if we can send the new version and raise an exception if we can\u0027t which we can return as a 400 out of the API as today. Similar to this:\n\nhttps://github.com/openstack/nova/blob/c966524672cedba5c1bc40b76daf0b3745dc0c1d/nova/compute/rpcapi.py#L1044","commit_id":"23e31b495e97f640cdb7c6427454d2e5d6aa187f"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2f9dfbe24d71760c0b58254527ba32e7629cfd24","unresolved":false,"context_lines":[{"line_number":171,"context_line":"Upgrade impact"},{"line_number":172,"context_line":"--------------"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"None"},{"line_number":175,"context_line":""},{"line_number":176,"context_line":"Implementation"},{"line_number":177,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_e261faaa","line":174,"updated":"2018-03-28 22:21:07.000000000","message":"This is where you should make a note that the compute API will still return a 400 for trying to abort a queued migration if the compute service that the instance is running on is too old.","commit_id":"23e31b495e97f640cdb7c6427454d2e5d6aa187f"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":67,"context_line":"  the current ``eventlet.spawn_n()`` + python ``Semaphore`` implementation."},{"line_number":68,"context_line":"  The size of the Thread Pool will be limited by"},{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that later. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_7774b536","line":70,"range":{"start_line":70,"start_character":2,"end_line":70,"end_character":6},"updated":"2018-04-11 17:04:22.000000000","message":"come","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":67,"context_line":"  the current ``eventlet.spawn_n()`` + python ``Semaphore`` implementation."},{"line_number":68,"context_line":"  The size of the Thread Pool will be limited by"},{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that later. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_3a0624d0","line":70,"range":{"start_line":70,"start_character":2,"end_line":70,"end_character":6},"in_reply_to":"bf659307_7774b536","updated":"2018-04-11 17:19:02.000000000","message":"comes :)","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that later. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* Add a new ``_waiting_live_migration`` variable to the ``ComputeManager``"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_5795f131","line":72,"range":{"start_line":72,"start_character":16,"end_line":72,"end_character":23},"updated":"2018-04-11 17:04:22.000000000","message":"s/comming//","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  ``CONF.max_concurrent_live_migrations``. When a live migration request"},{"line_number":70,"context_line":"  came in, we submit the ``_do_live_migration`` calls to the pool, and it"},{"line_number":71,"context_line":"  will return a ``Future`` object, we will use that later. If the pool is"},{"line_number":72,"context_line":"  full, the new comming request will be blocked and kept in ``queued``"},{"line_number":73,"context_line":"  status."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* Add a new ``_waiting_live_migration`` variable to the ``ComputeManager``"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_7a6a5c81","line":72,"range":{"start_line":72,"start_character":16,"end_line":72,"end_character":23},"in_reply_to":"bf659307_5795f131","updated":"2018-04-11 17:19:02.000000000","message":"I assume this was supposed to be \"incoming\" but removing it is also OK.","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":113,"context_line":"  live migration in ``queued`` status. If the microversion of the request"},{"line_number":114,"context_line":"  is equal or beyond the newly added microversion, API will check the"},{"line_number":115,"context_line":"  ``instance.host\u0027s`` nova-compute service version and make sure it is"},{"line_number":116,"context_line":"  new enough for the new support, if not, API will still return 400 as today."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":119,"context_line":"  shutting down. This can simply be done by calling"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_97014909","line":116,"updated":"2018-04-11 17:04:22.000000000","message":"This means a new RPC call right? Will this be a cast from the api to the compute, returning 202 to the client, or do we expect to make a call and report whether or not we were able to .cancel() the future? Or do we use instance actions (or something) to indicate that we tried and failed to cancel or something?","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"13f078808c77a3d3a97c0d8648aa67397a70cc37","unresolved":false,"context_lines":[{"line_number":113,"context_line":"  live migration in ``queued`` status. If the microversion of the request"},{"line_number":114,"context_line":"  is equal or beyond the newly added microversion, API will check the"},{"line_number":115,"context_line":"  ``instance.host\u0027s`` nova-compute service version and make sure it is"},{"line_number":116,"context_line":"  new enough for the new support, if not, API will still return 400 as today."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":119,"context_line":"  shutting down. This can simply be done by calling"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_3a15c49c","line":116,"in_reply_to":"bf659307_1a2a402b","updated":"2018-04-11 17:26:30.000000000","message":"Okay, so minimum for everyone, based on the version we\u0027re pinned to. We don\u0027t have the actual migration in the rpc layer there to check the state and react to the version, so I guess something has to change to make that work.\n\nBut that\u0027s fine. I wasn\u0027t thinking about re-using that method I guess.","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":113,"context_line":"  live migration in ``queued`` status. If the microversion of the request"},{"line_number":114,"context_line":"  is equal or beyond the newly added microversion, API will check the"},{"line_number":115,"context_line":"  ``instance.host\u0027s`` nova-compute service version and make sure it is"},{"line_number":116,"context_line":"  new enough for the new support, if not, API will still return 400 as today."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":119,"context_line":"  shutting down. This can simply be done by calling"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_1a2a402b","line":116,"in_reply_to":"bf659307_97014909","updated":"2018-04-11 17:19:02.000000000","message":"It\u0027s not a new RPC call. I think it would be based on what we did here for multiattach, which was based on what we did for tagged attach:\n\nhttps://github.com/openstack/nova/blob/c966524672cedba5c1bc40b76daf0b3745dc0c1d/nova/compute/rpcapi.py#L1044","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":119,"context_line":"  shutting down. This can simply be done by calling"},{"line_number":120,"context_line":"  ``ThreadPoolExecutor.shutdown(wait\u003dFalse)``."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Alternatives"},{"line_number":123,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_57f65165","line":120,"updated":"2018-04-11 17:04:22.000000000","message":"And do what in the cleanup? Mark the migration(s) as in cancelled state I assume?","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* We will also add a cleanup to the pool when the compute manager is"},{"line_number":119,"context_line":"  shutting down. This can simply be done by calling"},{"line_number":120,"context_line":"  ``ThreadPoolExecutor.shutdown(wait\u003dFalse)``."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Alternatives"},{"line_number":123,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_1aad609a","line":120,"in_reply_to":"bf659307_57f65165","updated":"2018-04-11 17:19:02.000000000","message":"I guess I didn\u0027t think this would be a callback cleanup method or anything, just that the cleanup_host() method in the compute manager would call shutdown(wait\u003dFalse) on the thread pool executor.\n\nLooking at http://pythonhosted.org/futures/#concurrent.futures.Executor.shutdown it doesn\u0027t return anything, so I\u0027m not what we can do with that.\n\nLooking at the Futures object though, http://pythonhosted.org/futures/#future-objects - it looks like we could add a done callback:\n\nhttp://pythonhosted.org/futures/#concurrent.futures.Future.add_done_callback\n\nWhich would update the migration status to cancelled, and then we could call cancel() on each future:\n\nhttp://pythonhosted.org/futures/#concurrent.futures.Future.cancel","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":"Compute API will still return 400 for trying abort a migration in"},{"line_number":178,"context_line":"queued state if the compute service that the instance is running on"},{"line_number":179,"context_line":"is too old."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"Implementation"},{"line_number":182,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_d786e1b9","line":179,"updated":"2018-04-11 17:04:22.000000000","message":"So we\u0027re going to look up the individual service and check its version, or use the minimum version bar?","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":"Compute API will still return 400 for trying abort a migration in"},{"line_number":178,"context_line":"queued state if the compute service that the instance is running on"},{"line_number":179,"context_line":"is too old."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"Implementation"},{"line_number":182,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_fa80acf9","line":179,"in_reply_to":"bf659307_d786e1b9","updated":"2018-04-11 17:19:02.000000000","message":"See above.","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e386f264a99e4e157e31fb3b93d26332168969d2","unresolved":false,"context_lines":[{"line_number":191,"context_line":"----------"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"* Create a new API microversion to allow abort live migrations in"},{"line_number":194,"context_line":"  ``queued`` status."},{"line_number":195,"context_line":"* Modify the Nova client to handle the new microversion."},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_374ddd99","line":194,"updated":"2018-04-11 17:04:22.000000000","message":"These could be broken up quite a bit I think, especially since you have it done more above.\n\n * Convert compute manager to queue migrations with threads/futures\n * Add an RPC call to cancel a migration in queued state\n * Add the REST api and microversion to let you actually do it\n * Modify the novaclient so we can test it","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a8cb2841cfc9321bf36b3c1575ea54a8031053d4","unresolved":false,"context_lines":[{"line_number":191,"context_line":"----------"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"* Create a new API microversion to allow abort live migrations in"},{"line_number":194,"context_line":"  ``queued`` status."},{"line_number":195,"context_line":"* Modify the Nova client to handle the new microversion."},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_3a906443","line":194,"in_reply_to":"bf659307_374ddd99","updated":"2018-04-11 17:19:02.000000000","message":"\u003e Add an RPC call to cancel a migration in queued state\n\nThat already exists, it\u0027s just that the API doesn\u0027t allow calling it unless the migration status is in \u0027running\u0027 status:\n\nhttps://github.com/openstack/nova/blob/c966524672cedba5c1bc40b76daf0b3745dc0c1d/nova/compute/api.py#L4225","commit_id":"442f66ad2daee928e9c6204a0d680f0bdae98146"}]}