)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"a0e08ff3b56321c98b81208a132d7b83a855c49d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"675fe9e0_757bc011","updated":"2026-04-27 19:34:29.000000000","message":"need to add more details","commit_id":"992173087e341295d7609ea4f6f0ab5b8fb98a48"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7a722433e46add8897f6370bfe4b8d8de569fc77","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"bb5fe195_09d849dd","updated":"2026-05-28 13:55:37.000000000","message":"Looks good, mostly okay with me, just another typo and a couple questions about the tracking dispatch logic.","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"4c517b14b77314362b8c52e9e50601a7590a7b93","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"2fd95c2c_bdaea327","updated":"2026-06-01 19:16:17.000000000","message":"This looks OK to me. Inline I commented a few spelling/grammar things I noticed while I was reading it.","commit_id":"8a1f54fc53b4fa0b864e6ed56881f4ab6b2d65ea"}],"specs/2026.2/approved/nova-services-graceful-shutdown-part2-task-tracking.rst":[{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"* Operators have no way to know which tasks are blocking shutdown, how long"},{"line_number":33,"context_line":"  they have been running, or which ones are still not completed when service"},{"line_number":34,"context_line":"  is shutdown due to timeout."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Use Cases"},{"line_number":37,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"23f40feb_3aabd9cd","line":34,"updated":"2026-05-27 18:11:45.000000000","message":"You might change or amend this to add \"if the timeout is not sufficient for some remaining tasks, we will just exit anyway without waiting\". The second bullet here seems to be about operator visibility (which is valid) but it\u0027s also the case that nova itself has no idea what is still waiting or not.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"* Operators have no way to know which tasks are blocking shutdown, how long"},{"line_number":33,"context_line":"  they have been running, or which ones are still not completed when service"},{"line_number":34,"context_line":"  is shutdown due to timeout."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Use Cases"},{"line_number":37,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"c355ac28_87f00665","line":34,"in_reply_to":"23f40feb_3aabd9cd","updated":"2026-05-27 19:05:18.000000000","message":"Done","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":52,"context_line":"consists of:"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"* Tasks will be tracked in a dictionary with info task name, request-id,"},{"line_number":55,"context_line":"  and instance UUID (if they are associated with instance)."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Once shutdown is initiated, a boolean flag ``_shutdown_in_progress`` will"},{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"eba456fc_af6a0680","line":55,"updated":"2026-05-27 18:11:45.000000000","message":"I wonder if it\u0027s reasonable to add a `start_time` and `estimated_deadline` or something to these task objects. While they may be hard-coded now, we may want to wait a long time for live migrations, but not wait a long time for an instance reboot, snapshot, etc.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"3a45972e978b51f287184df4616aba07781f58cd","unresolved":false,"context_lines":[{"line_number":52,"context_line":"consists of:"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"* Tasks will be tracked in a dictionary with info task name, request-id,"},{"line_number":55,"context_line":"  and instance UUID (if they are associated with instance)."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Once shutdown is initiated, a boolean flag ``_shutdown_in_progress`` will"},{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"14f69b6c_010df9ed","line":55,"in_reply_to":"820c0f12_eea9868e","updated":"2026-05-27 20:22:46.000000000","message":"I get your point now which make sense. I agree that can be improved/added later, for now, I will add the start_time.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"be097422475cc6a02e7bdbace0285ecafc7c4d6c","unresolved":true,"context_lines":[{"line_number":52,"context_line":"consists of:"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"* Tasks will be tracked in a dictionary with info task name, request-id,"},{"line_number":55,"context_line":"  and instance UUID (if they are associated with instance)."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Once shutdown is initiated, a boolean flag ``_shutdown_in_progress`` will"},{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"820c0f12_eea9868e","line":55,"in_reply_to":"c33514cd_2de1c522","updated":"2026-05-27 19:30:23.000000000","message":"Well, like I said.. I can imagine a future where the operator wants to say \"wait up to 60m for any live migrations, but snapshots are less important.. only wait 5m for those and if they\u0027re not done, abort\". Similarly, instance start/stop is very low importance, easy to resume later, etc.\n\nNot critical, I just think that like today, we have different parallelism numbers for starts, migrations, etc, and this is a similar thing. Some are very important and some are less so.\n\nIt\u0027s also easy to add later if we\u0027re not persisting these structures. At least a `start_time` would be useful I think so the operator can see \"okay, this snapshot has been going for four hours and it\u0027s the last thing being waited for, I\u0027m going to `kill -9`\".","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":true,"context_lines":[{"line_number":52,"context_line":"consists of:"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"* Tasks will be tracked in a dictionary with info task name, request-id,"},{"line_number":55,"context_line":"  and instance UUID (if they are associated with instance)."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Once shutdown is initiated, a boolean flag ``_shutdown_in_progress`` will"},{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"c33514cd_2de1c522","line":55,"in_reply_to":"eba456fc_af6a0680","updated":"2026-05-27 19:05:18.000000000","message":"Sure, but with estimated_deadline, what will be our action? do you mean to stop those tasks if they\u0027re not completed within that deadline? I think we should let them run until the manager shuts down the timeout. If they take more time than that, why?\n\nI mean, we can add start and end times to see how long each task took.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"},{"line_number":59,"context_line":"  in-progress tasks."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"* Once shutdown is initiated, it will stop submitting the new periodic tasks."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* The configurable ``manager_shutdown_timeout`` is the max time a service"},{"line_number":64,"context_line":"  manager will wait for tasks to finish."}],"source_content_type":"text/x-rst","patch_set":2,"id":"2ab392ca_679110b9","line":61,"range":{"start_line":61,"start_character":62,"end_line":61,"end_character":70},"updated":"2026-05-27 18:11:45.000000000","message":"Did you mean \"tasks\" here (i.e. not periodic, but new tasks like live migration)?\n\nStopping the periodics might also make sense, but it seemed like you meant new non-periodic tasks.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":false,"context_lines":[{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"},{"line_number":59,"context_line":"  in-progress tasks."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"* Once shutdown is initiated, it will stop submitting the new periodic tasks."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* The configurable ``manager_shutdown_timeout`` is the max time a service"},{"line_number":64,"context_line":"  manager will wait for tasks to finish."}],"source_content_type":"text/x-rst","patch_set":2,"id":"b9b493b7_b37441e1","line":61,"range":{"start_line":61,"start_character":62,"end_line":61,"end_character":70},"in_reply_to":"2ab392ca_679110b9","updated":"2026-05-27 19:05:18.000000000","message":"Yes, new non-periodic tasks will be handled by the RPC servers. Once shutdown is initiated, the first RPC server (the new tasks/normal operation RPC server) will be stopped, preventing it from accepting any new tasks. This bullet was mainly for the new periodic tasks, but let me clarify it for new non-periodic tasks also.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"be097422475cc6a02e7bdbace0285ecafc7c4d6c","unresolved":false,"context_lines":[{"line_number":58,"context_line":"  bet set to ``True`` which will signal service manager to start logging the"},{"line_number":59,"context_line":"  in-progress tasks."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"* Once shutdown is initiated, it will stop submitting the new periodic tasks."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* The configurable ``manager_shutdown_timeout`` is the max time a service"},{"line_number":64,"context_line":"  manager will wait for tasks to finish."}],"source_content_type":"text/x-rst","patch_set":2,"id":"8378f96c_8f01db3e","line":61,"range":{"start_line":61,"start_character":62,"end_line":61,"end_character":70},"in_reply_to":"b9b493b7_b37441e1","updated":"2026-05-27 19:30:23.000000000","message":"Ack, I saw the periodic bit below after I wrote this, so makes sense. But yeah, a few more words might be good.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":96,"context_line":"   operation completes so tracking the RPC call is not the correct way. For"},{"line_number":97,"context_line":"   these async tasks, tracking will check the actual task is completed via"},{"line_number":98,"context_line":"   future.add_done_callback() method which allow to attaches a callable"},{"line_number":99,"context_line":"   that will be executed automatically when a Future object is finished."},{"line_number":100,"context_line":"   Something like"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":".. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baa504e9_5798e58c","line":99,"updated":"2026-05-27 18:11:45.000000000","message":"I guess this is okay, but we could also just have a decorator that wraps the inner \"actual work\" functions right?","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"be097422475cc6a02e7bdbace0285ecafc7c4d6c","unresolved":false,"context_lines":[{"line_number":96,"context_line":"   operation completes so tracking the RPC call is not the correct way. For"},{"line_number":97,"context_line":"   these async tasks, tracking will check the actual task is completed via"},{"line_number":98,"context_line":"   future.add_done_callback() method which allow to attaches a callable"},{"line_number":99,"context_line":"   that will be executed automatically when a Future object is finished."},{"line_number":100,"context_line":"   Something like"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":".. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3b3a8439_002b2447","line":99,"in_reply_to":"0ac969f8_64ac0ffc","updated":"2026-05-27 19:30:23.000000000","message":"Acknowledged","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":true,"context_lines":[{"line_number":96,"context_line":"   operation completes so tracking the RPC call is not the correct way. For"},{"line_number":97,"context_line":"   these async tasks, tracking will check the actual task is completed via"},{"line_number":98,"context_line":"   future.add_done_callback() method which allow to attaches a callable"},{"line_number":99,"context_line":"   that will be executed automatically when a Future object is finished."},{"line_number":100,"context_line":"   Something like"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":".. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":2,"id":"0ac969f8_64ac0ffc","line":99,"in_reply_to":"baa504e9_5798e58c","updated":"2026-05-27 19:05:18.000000000","message":"yeah, that can also work. The only benefit of callbacks is readability: we can add start and end tracking methods in a single function instead of starting in the main function and finishing in an inner function executed by a thread.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":117,"context_line":"   ``reboot_instance()``, ``get_console_output()`` etc run synchronously."},{"line_number":118,"context_line":"   They perform all their work before the RPC request is returned. We have two"},{"line_number":119,"context_line":"   ways to track them. 1. Record the tasks by decorating the each RPC methods"},{"line_number":120,"context_line":"   will not be a good idea. We can have a commong wrapper who can record/track"},{"line_number":121,"context_line":"   these sync RPC requests."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   A new RPC endpoint wrapper will be added. When RPC dispatcher will send the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"debecfe1_dacc1d26","line":120,"range":{"start_line":120,"start_character":42,"end_line":120,"end_character":49},"updated":"2026-05-27 18:11:45.000000000","message":"\"common\"","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":false,"context_lines":[{"line_number":117,"context_line":"   ``reboot_instance()``, ``get_console_output()`` etc run synchronously."},{"line_number":118,"context_line":"   They perform all their work before the RPC request is returned. We have two"},{"line_number":119,"context_line":"   ways to track them. 1. Record the tasks by decorating the each RPC methods"},{"line_number":120,"context_line":"   will not be a good idea. We can have a commong wrapper who can record/track"},{"line_number":121,"context_line":"   these sync RPC requests."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"   A new RPC endpoint wrapper will be added. When RPC dispatcher will send the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"d8dc2ba9_b023dac9","line":120,"range":{"start_line":120,"start_character":42,"end_line":120,"end_character":49},"in_reply_to":"debecfe1_dacc1d26","updated":"2026-05-27 19:05:18.000000000","message":"Done","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                 finally:"},{"line_number":155,"context_line":"                     self._manager._finish_active_rpc_call(key)"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"             return _track_task"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Waiting logic"},{"line_number":160,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"b0cc44d8_bcf54af5","line":157,"updated":"2026-05-27 18:11:45.000000000","message":"Hmm, I don\u0027t understand your plan here. Does this thing wrap the entire manager and all the calls within?","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"040d4a77a90b782714598a95daa9f1ebc2a8ab3f","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                 finally:"},{"line_number":155,"context_line":"                     self._manager._finish_active_rpc_call(key)"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"             return _track_task"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Waiting logic"},{"line_number":160,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"4650e596_a8e0f0f0","line":157,"in_reply_to":"3377823e_a10b7f53","updated":"2026-05-27 19:42:09.000000000","message":"I forgot to mention here that it will exclude the async tasks to track as part of this wrapper, which are tracked in another way, L90. We need to hardcode the async tasks (like build) that are actually completed by another thread (spawn) so that this wrapper can exclude them and avoid any duplicate tracking (or false tracking of async tasks to mark complete as soon as RPC is returned).\n\nI will add that part.","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"3a45972e978b51f287184df4616aba07781f58cd","unresolved":false,"context_lines":[{"line_number":154,"context_line":"                 finally:"},{"line_number":155,"context_line":"                     self._manager._finish_active_rpc_call(key)"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"             return _track_task"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Waiting logic"},{"line_number":160,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"69e3385c_9ac355c1","line":157,"in_reply_to":"4650e596_a8e0f0f0","updated":"2026-05-27 20:22:46.000000000","message":"Done","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"be097422475cc6a02e7bdbace0285ecafc7c4d6c","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                 finally:"},{"line_number":155,"context_line":"                     self._manager._finish_active_rpc_call(key)"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"             return _track_task"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Waiting logic"},{"line_number":160,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3377823e_a10b7f53","line":157,"in_reply_to":"63654993_7fda0025","updated":"2026-05-27 19:30:23.000000000","message":"Hmm, okay I guess I wasn\u0027t thinking this would apply to literally everything in the manager. Not everything in the manager is an RPC call, although I guess if you do this at the right layer we\u0027ll only do the start task part when an actual RPC is done...\n\nSo this will wrap everything automatically, and then anything that spawns and returns (like build) will create _another_ task for the actual work and the automatic one will be recorded, but finish very fast?","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                 finally:"},{"line_number":155,"context_line":"                     self._manager._finish_active_rpc_call(key)"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"             return _track_task"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Waiting logic"},{"line_number":160,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"63654993_7fda0025","line":157,"in_reply_to":"b0cc44d8_bcf54af5","updated":"2026-05-27 19:05:18.000000000","message":"Yeah, it will wrap the entire manager. As you know, service.py initializes the manager and passes it to the RPC server; that manager instance will be wrapped with the TrackingEndpointWrapper. The RPC dispatcher then routes the call to this wrapper endpoint, which tracks the task and calls the actual manager method to perform the operation.\n\nsomething like:\n\nExisting:\n\n        endpoints \u003d [\n            self.manager,\n            baserpc.BaseRPCAPI(self.manager.service_name, self.backdoor_port)\n        ]\n\nto:\n\n        manager_endpoint \u003d _TrackingEndpointWrapper(self.manager)\n        endpoints \u003d [\n            manager_endpoint,\n            baserpc.BaseRPCAPI(self.manager.service_name, self.backdoor_port)\n        ]","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"810855ebe1b1b3f7d163fdb1c738ed97aca0d7f3","unresolved":true,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"* Add common task tracking framework methods."},{"line_number":270,"context_line":"* Audit the async tasks and track them as per the async tasks tracking"},{"line_number":271,"context_line":"  mechanishm."},{"line_number":272,"context_line":"* Add functional tests."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":2,"id":"57035256_38905934","line":271,"range":{"start_line":271,"start_character":2,"end_line":271,"end_character":12},"updated":"2026-05-27 18:11:45.000000000","message":"\"mechanism\"","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"63d7fa8d0dc8efaff4966ae7ee9f85a893f20821","unresolved":false,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"* Add common task tracking framework methods."},{"line_number":270,"context_line":"* Audit the async tasks and track them as per the async tasks tracking"},{"line_number":271,"context_line":"  mechanishm."},{"line_number":272,"context_line":"* Add functional tests."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":2,"id":"13696334_7ca3e9bf","line":271,"range":{"start_line":271,"start_character":2,"end_line":271,"end_character":12},"in_reply_to":"57035256_38905934","updated":"2026-05-27 19:05:18.000000000","message":"Done","commit_id":"b0ec3d7c0f21c93390948186d04d2a415d69016d"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"4c517b14b77314362b8c52e9e50601a7590a7b93","unresolved":true,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This spec implements Spec 2 of the `nova-services-graceful-shutdown`_ backlog"},{"line_number":14,"context_line":"spec for the Nova service. Spec 1 introduced a second RPC server in compute"},{"line_number":15,"context_line":"service to separate new requets from in-progress requests and time"},{"line_number":16,"context_line":"based wait in services manager to finish the in-progress tasks. This spec"},{"line_number":17,"context_line":"replaces the time based wait with a proper task-tracking system that track"},{"line_number":18,"context_line":"the in-progress tasks and their completion with detailed logging."}],"source_content_type":"text/x-rst","patch_set":3,"id":"03c368c3_e2d0f980","line":15,"range":{"start_line":15,"start_character":24,"end_line":15,"end_character":31},"updated":"2026-06-01 19:16:17.000000000","message":"requests","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"4c517b14b77314362b8c52e9e50601a7590a7b93","unresolved":true,"context_lines":[{"line_number":44,"context_line":"  when shutdown is initiated, and which ones are still not completed when"},{"line_number":45,"context_line":"  service is shutdown due to timeout. This will help them to estimate the"},{"line_number":46,"context_line":"  graceful_shutdown_timeout in their deployment and also re-initiate the"},{"line_number":47,"context_line":"  non-comleted operations."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Proposed change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"0847a428_c0044f92","line":47,"range":{"start_line":47,"start_character":6,"end_line":47,"end_character":14},"updated":"2026-06-01 19:16:17.000000000","message":"completed","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7a722433e46add8897f6370bfe4b8d8de569fc77","unresolved":true,"context_lines":[{"line_number":130,"context_line":"   This new wrapper will not wrap the below methods:"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    1. Async tasks (mentioned in above section) methods:"},{"line_number":133,"context_line":"       The async tasks are tracked in different way. We will excelude those"},{"line_number":134,"context_line":"       in this wrapper so that those are not tracked twice or marked as"},{"line_number":135,"context_line":"       false completed based on RPC return."},{"line_number":136,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"949f8d8e_9a5af135","line":133,"range":{"start_line":133,"start_character":61,"end_line":133,"end_character":69},"updated":"2026-05-28 13:55:37.000000000","message":"\"exclude\"","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5636886e59a53525b8ac5213887604a08f51be9d","unresolved":false,"context_lines":[{"line_number":130,"context_line":"   This new wrapper will not wrap the below methods:"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    1. Async tasks (mentioned in above section) methods:"},{"line_number":133,"context_line":"       The async tasks are tracked in different way. We will excelude those"},{"line_number":134,"context_line":"       in this wrapper so that those are not tracked twice or marked as"},{"line_number":135,"context_line":"       false completed based on RPC return."},{"line_number":136,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"5c607918_7c784fb8","line":133,"range":{"start_line":133,"start_character":61,"end_line":133,"end_character":69},"in_reply_to":"949f8d8e_9a5af135","updated":"2026-05-28 18:41:15.000000000","message":"Done","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7a722433e46add8897f6370bfe4b8d8de569fc77","unresolved":true,"context_lines":[{"line_number":146,"context_line":"         def __getattr__(self, name):"},{"line_number":147,"context_line":"             attr \u003d getattr(self._manager, name)"},{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"41cf94cb_72ca84f7","line":149,"updated":"2026-05-28 13:55:37.000000000","message":"I think this is good, but this should never happen right? The RPC server should never be calling `manager._foo()` because no RPC request will ever arrive for it, correct?\n\n(That makes me realize that maybe the RPC server should enforce that no remote client tries to call dunder methods on a manager. If any malicious actor gets on the RPC bus all bets are off, but at least blocking calling obviously-private methods might be prudent. Unrelated to this work though of course. We also have some non-RPC methods that do not start with `_` in the compute manager, which should probably be renamed so they don\u0027t look like RPC-able methods.)","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5636886e59a53525b8ac5213887604a08f51be9d","unresolved":true,"context_lines":[{"line_number":146,"context_line":"         def __getattr__(self, name):"},{"line_number":147,"context_line":"             attr \u003d getattr(self._manager, name)"},{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"96677d0c_2b601200","line":149,"in_reply_to":"41cf94cb_72ca84f7","updated":"2026-05-28 18:41:15.000000000","message":"Yeah, we do not have such a case. I will remove this protection. But that is a good point that we have a lot of non-RPC methods not declared with ``_``. I am not sure all are internal-calling ones, but agree to differentiate them from RPC methods. I will check to do that, but yes, as a separate effort.","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b7b85ba7bbdd7e6508030758786af696dab92351","unresolved":false,"context_lines":[{"line_number":146,"context_line":"         def __getattr__(self, name):"},{"line_number":147,"context_line":"             attr \u003d getattr(self._manager, name)"},{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"d2fe51bb_54502e04","line":149,"in_reply_to":"96677d0c_2b601200","updated":"2026-05-28 19:19:37.000000000","message":"Done","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7a722433e46add8897f6370bfe4b8d8de569fc77","unresolved":true,"context_lines":[{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"             @functools.wraps(attr)"},{"line_number":154,"context_line":"             def _track_task(* args, ** kwargs):"}],"source_content_type":"text/x-rst","patch_set":3,"id":"37d40026_c4930191","line":151,"updated":"2026-05-28 13:55:37.000000000","message":"And how will these get here? Some sort of exclusionary decorator that says \"this just starts an async task so don\u0027t track it\"?\n\nI wonder if we should make this more generic and just call these \"untracked methods.\" Just thinking if there is a case where we might want to mark something as untracked because it is not long-running, or is fully interruptible, etc. Like maybe `cache_images()` or `get_instance_diagnostics()` or even `get_vnc_console()`. Those are all things we wouldn\u0027t want to prevent us from shutting down, I think.","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5636886e59a53525b8ac5213887604a08f51be9d","unresolved":true,"context_lines":[{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"             @functools.wraps(attr)"},{"line_number":154,"context_line":"             def _track_task(* args, ** kwargs):"}],"source_content_type":"text/x-rst","patch_set":3,"id":"71e6d177_d6565617","line":151,"in_reply_to":"37d40026_c4930191","updated":"2026-05-28 18:41:15.000000000","message":"I was thinking of hard-coding it (_ASYNC_TRACKED_METHODS \u003d [\u0027live_migration\u0027, \u0027build_and_run_instance\u0027......]), but I like the idea of marking it as generic so that we can also exclude interruptable tasks (along with log-running tasks) which can eventually be modified/extended over time without changing much logic.","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b7b85ba7bbdd7e6508030758786af696dab92351","unresolved":false,"context_lines":[{"line_number":148,"context_line":"            if not callable(attr) or name.startswith(\u0027_\u0027):"},{"line_number":149,"context_line":"                return attr"},{"line_number":150,"context_line":"            if name in self._manager._ASYNC_TRACKED_METHODS:"},{"line_number":151,"context_line":"                return attr"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"             @functools.wraps(attr)"},{"line_number":154,"context_line":"             def _track_task(* args, ** kwargs):"}],"source_content_type":"text/x-rst","patch_set":3,"id":"32b5645d_d0c9af44","line":151,"in_reply_to":"71e6d177_d6565617","updated":"2026-05-28 19:19:37.000000000","message":"Done","commit_id":"c99f03de0ea3771c2ceb7290ea2eb777b01ad914"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"4c517b14b77314362b8c52e9e50601a7590a7b93","unresolved":true,"context_lines":[{"line_number":84,"context_line":"Nova has the two categories of tasks which need to be tracked separately"},{"line_number":85,"context_line":"because of their different execution models."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":".. note:: These two catagories are not the RPC call vs RPC cast. Those can"},{"line_number":88,"context_line":"  be tracked once the RPC method is called and finish its execution."},{"line_number":89,"context_line":"  RPC cast does not return any response and RPC call complete its execution"},{"line_number":90,"context_line":"  and return response via the reply queue. The reply queues are active during"}],"source_content_type":"text/x-rst","patch_set":4,"id":"4435ba6d_abea2c4b","line":87,"range":{"start_line":87,"start_character":20,"end_line":87,"end_character":30},"updated":"2026-06-01 19:16:17.000000000","message":"categories","commit_id":"8a1f54fc53b4fa0b864e6ed56881f4ab6b2d65ea"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"4c517b14b77314362b8c52e9e50601a7590a7b93","unresolved":true,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"   A new RPC endpoint wrapper will be added. When RPC dispatcher will send the"},{"line_number":127,"context_line":"   RPC message to endpoint (service manager), the endpoint wrapper can record"},{"line_number":128,"context_line":"   the tasks before calling the manager RPC method. That will looks like"},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"   Not all the manager methods will be tracked by this wrapper. A few examples"},{"line_number":131,"context_line":"   of such methods are long-running tasks in the background, tasks which are ok"}],"source_content_type":"text/x-rst","patch_set":4,"id":"47b5dff3_1f2127d4","line":128,"range":{"start_line":128,"start_character":52,"end_line":128,"end_character":72},"updated":"2026-06-01 19:16:17.000000000","message":"? unfinished sentence","commit_id":"8a1f54fc53b4fa0b864e6ed56881f4ab6b2d65ea"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"d06622bfa728b3b3f156c49d81d50e73b6ef881e","unresolved":true,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":".. code-block:: python"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    def untracked_rpc_method(fn):"},{"line_number":139,"context_line":"        fn._skip_rpc_tracking \u003d True"},{"line_number":140,"context_line":"        return fn"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"deaa23a7_f2795888","line":138,"updated":"2026-05-28 19:42:10.000000000","message":"We can argue about the names later, but yeah I like this better than a static list I think.","commit_id":"8a1f54fc53b4fa0b864e6ed56881f4ab6b2d65ea"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"2a670f2268c4383ac8069006dd8f449a3eeec706","unresolved":true,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":".. code-block:: python"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    def untracked_rpc_method(fn):"},{"line_number":139,"context_line":"        fn._skip_rpc_tracking \u003d True"},{"line_number":140,"context_line":"        return fn"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"6ed66cd5_bdb38832","line":138,"in_reply_to":"deaa23a7_f2795888","updated":"2026-05-29 02:25:06.000000000","message":"Ack. will try with better name but yes we can figure out it during implementation.","commit_id":"8a1f54fc53b4fa0b864e6ed56881f4ab6b2d65ea"}]}