)]}'
{"specs/newton/approved/recover-deleting-state.rst":[{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":56,"context_line":"host. This code is ran periodically and checks for existing VMs in DELETED"},{"line_number":57,"context_line":"state that has been deleted long enough ago."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"I want to propose adding similar periodic check for the DELETING status,"},{"line_number":60,"context_line":"together with adding information on when the status was changed to DELETING to"},{"line_number":61,"context_line":"the system metadata. This way we can tell that a given instance is stuck for a"},{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_916f46fb","line":59,"range":{"start_line":59,"start_character":18,"end_line":59,"end_character":47},"updated":"2016-02-23 21:58:03.000000000","message":"Perhaps we just extend this one to cover both cases? I think that having multiple config options for \"finish deleting stuff\" is likely to be confusing.\n\nhttps://github.com/openstack/nova/blob/master/nova/compute/manager.py#L6629-L6657","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":56,"context_line":"host. This code is ran periodically and checks for existing VMs in DELETED"},{"line_number":57,"context_line":"state that has been deleted long enough ago."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"I want to propose adding similar periodic check for the DELETING status,"},{"line_number":60,"context_line":"together with adding information on when the status was changed to DELETING to"},{"line_number":61,"context_line":"the system metadata. This way we can tell that a given instance is stuck for a"},{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_0a63acb2","line":59,"range":{"start_line":59,"start_character":18,"end_line":59,"end_character":47},"in_reply_to":"bae84128_916f46fb","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"40a2b42198806529363c4bee21028d69d9cc05f3","unresolved":false,"context_lines":[{"line_number":56,"context_line":"host. This code is ran periodically and checks for existing VMs in DELETED"},{"line_number":57,"context_line":"state that has been deleted long enough ago."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"I want to propose adding similar periodic check for the DELETING status,"},{"line_number":60,"context_line":"together with adding information on when the status was changed to DELETING to"},{"line_number":61,"context_line":"the system metadata. This way we can tell that a given instance is stuck for a"},{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_b57a1d22","line":59,"range":{"start_line":59,"start_character":18,"end_line":59,"end_character":47},"in_reply_to":"bae84128_916f46fb","updated":"2016-02-24 09:26:43.000000000","message":"That looks good, I didn\u0027t see that task. Thanks!","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":60,"context_line":"together with adding information on when the status was changed to DELETING to"},{"line_number":61,"context_line":"the system metadata. This way we can tell that a given instance is stuck for a"},{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"},{"line_number":63,"context_line":"in the rabbit queue or pending tasks."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_71bf2261","line":63,"updated":"2016-02-23 21:58:03.000000000","message":"I don\u0027t think we can really know that it\u0027s safe based on the timestamp in system_metadata, without a ttl on the message itself. Now, delete is special, and I think it\u0027s fine to go ahead and do this for it, but I wanted to make a point of saying that I don\u0027t think it\u0027s safe to do this for, say, volume_attach, unless we have the message TTL thing worked out.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":60,"context_line":"together with adding information on when the status was changed to DELETING to"},{"line_number":61,"context_line":"the system metadata. This way we can tell that a given instance is stuck for a"},{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"},{"line_number":63,"context_line":"in the rabbit queue or pending tasks."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_6a5870fb","line":63,"in_reply_to":"bae84128_71bf2261","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"},{"line_number":63,"context_line":"in the rabbit queue or pending tasks."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Alternatives"},{"line_number":67,"context_line":"------------"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_fc44a94a","line":65,"updated":"2016-02-23 21:58:03.000000000","message":"I think it\u0027s also important to state, somewhere above, the following:\n\nWe can\u0027t use updated_at, because something could be changing that value periodically such that we never reach enough elapsed time to reconsider the delete. We can\u0027t use deleted_at because we only set that once the delete actually happens, which means it\u0027ll never be set until we\u0027re past the outage interval we\u0027re trying to recover from here, which is that the initial delete never made it to compute.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":62,"context_line":"long time, and retry its deletion, without having to worry about messages stuck"},{"line_number":63,"context_line":"in the rabbit queue or pending tasks."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Alternatives"},{"line_number":67,"context_line":"------------"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_aa72787a","line":65,"in_reply_to":"bae84128_fc44a94a","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":71,"context_line":"somehow synchronize it with the database when there were any transient"},{"line_number":72,"context_line":"communication failures. This would solve a wider class of problems, as it will"},{"line_number":73,"context_line":"also help with migration and resizing. However, it\u0027s not yet clear how it would"},{"line_number":74,"context_line":"work exactly, and it requires a lot of work (and time)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_7c92594d","line":74,"updated":"2016-02-23 21:58:03.000000000","message":"I\u0027m not sure the above is really an alternative because it involves a bunch of things that are a looong way off.\n\nHowever, what about instead of this periodic scanning, we provide for letting the user actually issue a delete N minutes after the first one? That would mean that even if the thing is in deleting state, after ten (or thirty, or whatever) minutes, we allow them to re-issue a delete. If we lost the first one, that would let them do it again.\n\nYou could even set a sysmeta item on the instance once the compute starts the delete, and only let them retry a delete if they\u0027re in deleting state and that key is not set within the throttling interval. I think that would achieve the desired result, without adding more polling.\n\nJust something to think about. Maybe some other people (specifically API people) will comment on which they prefer.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":71,"context_line":"somehow synchronize it with the database when there were any transient"},{"line_number":72,"context_line":"communication failures. This would solve a wider class of problems, as it will"},{"line_number":73,"context_line":"also help with migration and resizing. However, it\u0027s not yet clear how it would"},{"line_number":74,"context_line":"work exactly, and it requires a lot of work (and time)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_6a0fd0e1","line":74,"in_reply_to":"bae84128_5563d141","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":71,"context_line":"somehow synchronize it with the database when there were any transient"},{"line_number":72,"context_line":"communication failures. This would solve a wider class of problems, as it will"},{"line_number":73,"context_line":"also help with migration and resizing. However, it\u0027s not yet clear how it would"},{"line_number":74,"context_line":"work exactly, and it requires a lot of work (and time)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_2a05c8c1","line":74,"in_reply_to":"bae84128_7c92594d","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"40a2b42198806529363c4bee21028d69d9cc05f3","unresolved":false,"context_lines":[{"line_number":71,"context_line":"somehow synchronize it with the database when there were any transient"},{"line_number":72,"context_line":"communication failures. This would solve a wider class of problems, as it will"},{"line_number":73,"context_line":"also help with migration and resizing. However, it\u0027s not yet clear how it would"},{"line_number":74,"context_line":"work exactly, and it requires a lot of work (and time)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_5563d141","line":74,"in_reply_to":"bae84128_7c92594d","updated":"2016-02-24 09:26:43.000000000","message":"That actually sounds much better than an automated cleanup, however, I think it would be necessary to clearly signal to the users that they can re-try the delete. The most obvious way to do that would be to add a new status, such as DELETE_FAILED or DELETE_TIMEOUT or maybe DELETE_STALLED, but that would require database migrations, AFAIK.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":117,"context_line":"There will be an additional key updated in the system metadata when it is"},{"line_number":118,"context_line":"being put into DELETING state."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"There will be a periodic task of the resource tracker on the compute with a"},{"line_number":121,"context_line":"loop, that checks all instances in DELETING state for the system metadata key,"},{"line_number":122,"context_line":"and if it matches the criteria, retries the delete."},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_dc6605ec","line":120,"range":{"start_line":120,"start_character":30,"end_line":120,"end_character":53},"updated":"2016-02-23 21:58:03.000000000","message":"I don\u0027t think this is accurate. The task would be on compute manager, not the resource tracker.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":117,"context_line":"There will be an additional key updated in the system metadata when it is"},{"line_number":118,"context_line":"being put into DELETING state."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"There will be a periodic task of the resource tracker on the compute with a"},{"line_number":121,"context_line":"loop, that checks all instances in DELETING state for the system metadata key,"},{"line_number":122,"context_line":"and if it matches the criteria, retries the delete."},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_8af69cd5","line":120,"range":{"start_line":120,"start_character":30,"end_line":120,"end_character":53},"in_reply_to":"bae84128_dc6605ec","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"There will be a periodic task of the resource tracker on the compute with a"},{"line_number":121,"context_line":"loop, that checks all instances in DELETING state for the system metadata key,"},{"line_number":122,"context_line":"and if it matches the criteria, retries the delete."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_7c853958","line":122,"updated":"2016-02-23 21:58:03.000000000","message":"All instances *on that host*, in deleting state, with that key set.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"There will be a periodic task of the resource tracker on the compute with a"},{"line_number":121,"context_line":"loop, that checks all instances in DELETING state for the system metadata key,"},{"line_number":122,"context_line":"and if it matches the criteria, retries the delete."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_ea1ac016","line":122,"in_reply_to":"bae84128_7c853958","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":125,"context_line":"Other deployer impact"},{"line_number":126,"context_line":"---------------------"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"There will need to be a new configuration options"},{"line_number":129,"context_line":"``stuck_deleting_instance_poll_interval`` for the cleanup task interval, and"},{"line_number":130,"context_line":"``stuck_deleting_instance_timeout`` for specifying the timeout after which the"},{"line_number":131,"context_line":"instance in DELETING state is re-deleted."}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_1c8b4d69","line":128,"range":{"start_line":128,"start_character":22,"end_line":128,"end_character":23},"updated":"2016-02-23 21:58:03.000000000","message":"s/a//","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":125,"context_line":"Other deployer impact"},{"line_number":126,"context_line":"---------------------"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"There will need to be a new configuration options"},{"line_number":129,"context_line":"``stuck_deleting_instance_poll_interval`` for the cleanup task interval, and"},{"line_number":130,"context_line":"``stuck_deleting_instance_timeout`` for specifying the timeout after which the"},{"line_number":131,"context_line":"instance in DELETING state is re-deleted."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_0ad8cc52","line":128,"range":{"start_line":128,"start_character":22,"end_line":128,"end_character":23},"in_reply_to":"bae84128_1c8b4d69","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"There will need to be a new configuration options"},{"line_number":129,"context_line":"``stuck_deleting_instance_poll_interval`` for the cleanup task interval, and"},{"line_number":130,"context_line":"``stuck_deleting_instance_timeout`` for specifying the timeout after which the"},{"line_number":131,"context_line":"instance in DELETING state is re-deleted."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_bc406108","line":130,"updated":"2016-02-23 21:58:03.000000000","message":"I think we should not have the poll interval, and either tack this into the existing delete cleanup task, or at least use the same interval. I think they\u0027re basically the same: \"how often I want you to look for deleted crap that needs re-deleting.\"\n\nI\u0027m not sure I think we need the timeout to be configurable either. We might be able to repurpose a config option we already have, and/or just choose a sane default, potentially based off of the scan interval. Like maybe 10*scan_interval or something.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"There will need to be a new configuration options"},{"line_number":129,"context_line":"``stuck_deleting_instance_poll_interval`` for the cleanup task interval, and"},{"line_number":130,"context_line":"``stuck_deleting_instance_timeout`` for specifying the timeout after which the"},{"line_number":131,"context_line":"instance in DELETING state is re-deleted."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_8acb3c72","line":130,"in_reply_to":"bae84128_bc406108","updated":"2016-03-07 15:32:14.000000000","message":"I left the second option for now, because I think we need more discussion here. It is my impression that the optimal timeout will depend on the size of the deployment and how \"busy\" the queue is?","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"04117efcecc90da0398da110fb05c34c6d1c10c2","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  system metadata."},{"line_number":166,"context_line":"* Refactor the code that does the cleanup at startup into a separate function."},{"line_number":167,"context_line":"* Add an ``timeout`` parameter to that function, and a check that only retries"},{"line_number":168,"context_line":"  deletion of the instances if they were set to DELETING longer ago that that."},{"line_number":169,"context_line":"* Add a new periodic task."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_bcafc17b","line":168,"updated":"2016-02-23 21:58:03.000000000","message":"Do we need the thing to be called on startup if we have the periodic? I mean, AFAIK, the delete message could be waiting for us in the queue when we restart such that we\u0027d process it normally. If not, then we\u0027ll do it when we run the periodic.","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  system metadata."},{"line_number":166,"context_line":"* Refactor the code that does the cleanup at startup into a separate function."},{"line_number":167,"context_line":"* Add an ``timeout`` parameter to that function, and a check that only retries"},{"line_number":168,"context_line":"  deletion of the instances if they were set to DELETING longer ago that that."},{"line_number":169,"context_line":"* Add a new periodic task."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_6aab508a","line":168,"in_reply_to":"bae84128_5598f11b","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"8700fd623684bdcdf31accfabb4440cba3883514","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  system metadata."},{"line_number":166,"context_line":"* Refactor the code that does the cleanup at startup into a separate function."},{"line_number":167,"context_line":"* Add an ``timeout`` parameter to that function, and a check that only retries"},{"line_number":168,"context_line":"  deletion of the instances if they were set to DELETING longer ago that that."},{"line_number":169,"context_line":"* Add a new periodic task."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aef4532_0aa60cc1","line":168,"in_reply_to":"bae84128_bcafc17b","updated":"2016-03-07 15:32:14.000000000","message":"Done","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":8648,"name":"Radomir Dopieralski","email":"openstack@dopieralski.pl","username":"thesheep"},"change_message_id":"40a2b42198806529363c4bee21028d69d9cc05f3","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  system metadata."},{"line_number":166,"context_line":"* Refactor the code that does the cleanup at startup into a separate function."},{"line_number":167,"context_line":"* Add an ``timeout`` parameter to that function, and a check that only retries"},{"line_number":168,"context_line":"  deletion of the instances if they were set to DELETING longer ago that that."},{"line_number":169,"context_line":"* Add a new periodic task."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bae84128_5598f11b","line":168,"in_reply_to":"bae84128_bcafc17b","updated":"2016-02-24 09:26:43.000000000","message":"Good point. Better to have it only in one place. Then again, at startup we don\u0027t need to respect the timeout?","commit_id":"491129e130a32bc6a745f67e2baa9943158cf0f9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":56,"context_line":"host. This code is ran periodically and checks for existing VMs in ``DELETED``"},{"line_number":57,"context_line":"state that has been deleted long enough ago."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"I want to propose extending the ``_run_pending_deletes`` periodic task so that"},{"line_number":60,"context_line":"it handles also instances that are in ``DELETING`` state for too long."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"To do that, we will need to add information on when the status was changed to"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_aa13710c","line":59,"updated":"2016-05-13 01:00:19.000000000","message":"Have you seen this change?\n\nhttps://review.openstack.org/#/c/294491/","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":84,"context_line":"Alternatives"},{"line_number":85,"context_line":"------------"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"There is some work on making the compute service hold its own local state. It"},{"line_number":88,"context_line":"would then be able to keep track of the state of instances on its own, and"},{"line_number":89,"context_line":"somehow synchronize it with the database when there were any transient"},{"line_number":90,"context_line":"communication failures. This would solve a wider class of problems, as it will"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_aaf7f1f3","line":87,"updated":"2016-05-13 01:00:19.000000000","message":"Do you have a reference to this work? Another spec, mailing list discussion, etc?","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":120,"context_line":"Security impact"},{"line_number":121,"context_line":"---------------"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"None. The additional action being executed is already being executed on restart"},{"line_number":124,"context_line":"anyways."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_6ad4e946","line":123,"range":{"start_line":123,"start_character":72,"end_line":123,"end_character":79},"updated":"2016-05-13 01:00:19.000000000","message":"nova-compute service restart","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":152,"context_line":"---------------------"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"There will need to be a new configuration option"},{"line_number":155,"context_line":"``stuck_deleting_instance_timeout`` for specifying the timeout after which the"},{"line_number":156,"context_line":"instance in ``DELETING`` state is re-deleted."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_2abb0110","line":155,"range":{"start_line":155,"start_character":2,"end_line":155,"end_character":33},"updated":"2016-05-13 01:00:19.000000000","message":"Which is then compared to the time set in system_metadata?","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":171,"context_line":"Assignee(s)"},{"line_number":172,"context_line":"-----------"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"Who is leading the writing of the code? Or is this a blueprint where you\u0027re"},{"line_number":175,"context_line":"throwing it out there to see who picks it up?"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"If more than one person is working on the implementation, please designate the"},{"line_number":178,"context_line":"primary author and contact."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"Primary assignee:"},{"line_number":181,"context_line":"    thesheep"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_6a3b499d","line":178,"range":{"start_line":174,"start_character":0,"end_line":178,"end_character":27},"updated":"2016-05-13 01:00:19.000000000","message":"You can remove this template text.","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":190,"context_line":"  into system meta-data."},{"line_number":191,"context_line":"* Add tests that check that the date is properly saved."},{"line_number":192,"context_line":"* Refactor the code that does the cleanup at startup into a separate function."},{"line_number":193,"context_line":"* Add an ``timeout`` parameter to that function, and a check that only retries"},{"line_number":194,"context_line":"  deletion of the instances if they were set to ``DELETING`` longer ago that"},{"line_number":195,"context_line":"  that."},{"line_number":196,"context_line":"* Add tests that check that the timeout parameter is respected."}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_2a52e1dd","line":193,"range":{"start_line":193,"start_character":11,"end_line":193,"end_character":18},"updated":"2016-05-13 01:00:19.000000000","message":"Which defaults to CONF.stuck_deleting_instance_timeout?","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":195,"context_line":"  that."},{"line_number":196,"context_line":"* Add tests that check that the timeout parameter is respected."},{"line_number":197,"context_line":"* Add a call to that function to the ``_run_pending_deletes`` task."},{"line_number":198,"context_line":"* Remove the call to that function from the manager init."},{"line_number":199,"context_line":""},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_0a727d7c","line":198,"updated":"2016-05-13 01:00:19.000000000","message":"But only if _run_pending_deletes will run, right? Because it is possible to disable periodic tasks via config options.","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fc2c0592c65fe5c743d864a52465593d5e549bb2","unresolved":false,"context_lines":[{"line_number":233,"context_line":""},{"line_number":234,"context_line":"   * - Release Name"},{"line_number":235,"context_line":"     - Description"},{"line_number":236,"context_line":"   * - Mitaka"},{"line_number":237,"context_line":"     - Introduced"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_ea6e1995","line":236,"range":{"start_line":236,"start_character":7,"end_line":236,"end_character":13},"updated":"2016-05-13 01:00:19.000000000","message":"Newton","commit_id":"a324fded38104d9d9925c5337e6ab99d31fae86b"}]}