)]}'
{"specs/ocata/approved/cleanup-running-shelved-instances.rst":[{"author":{"_account_id":20424,"name":"Eyal Posener","email":"eyal@stratoscale.com","username":"eyal"},"change_message_id":"a98d2b92c4ea0f4187055977bfc67ce61e184c23","unresolved":false,"context_lines":[{"line_number":49,"context_line":"nova-compute already have a mechanism of cleanup of running-deleted instances,"},{"line_number":50,"context_line":"controlled by the running_deleted_instance_action configuration parameter."},{"line_number":51,"context_line":"This suggests to add a configuration parameter - cleanup_shelved_instances,"},{"line_number":52,"context_line":"that will make the same mechaism cleaning-up shelved-offloaeded instances on"},{"line_number":53,"context_line":"the same loop."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a8ec9b2_2e0ab78a","line":52,"range":{"start_line":52,"start_character":24,"end_line":52,"end_character":32},"updated":"2016-09-21 10:59:19.000000000","message":"mechanism","commit_id":"fb134dc99eb117427412ce47efb6e8bef17c43b7"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"ccef396a27704cc1b3fbd5a073778bf2c231d311","unresolved":false,"context_lines":[{"line_number":40,"context_line":"The user then sees it, and re-tries the unshelving. The scheduler returns a"},{"line_number":41,"context_line":"different node, and the spawning is successful. We now have 2 instances of the"},{"line_number":42,"context_line":"same VM, on different nodes, writing to the same volume, and a data-corruption"},{"line_number":43,"context_line":"occurs."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa6399be_ff6a7e19","line":43,"updated":"2016-10-12 00:07:07.000000000","message":"So, why just offloaded though? Shouldn\u0027t this be any shelved instance which is still running? I wouldn\u0027t think a shelved instance should be executing at all (although it might still have files cached on the hypervisor).","commit_id":"8848f73485463d89ca5531cd5b6b2b215731398c"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"ebec54f0fd709a255746b38f75801343546b24a6","unresolved":false,"context_lines":[{"line_number":40,"context_line":"The user then sees it, and re-tries the unshelving. The scheduler returns a"},{"line_number":41,"context_line":"different node, and the spawning is successful. We now have 2 instances of the"},{"line_number":42,"context_line":"same VM, on different nodes, writing to the same volume, and a data-corruption"},{"line_number":43,"context_line":"occurs."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa6399be_da20684c","line":43,"in_reply_to":"fa6399be_ff6a7e19","updated":"2016-10-12 00:27:57.000000000","message":"sure thing. edited to include shelved as well","commit_id":"8848f73485463d89ca5531cd5b6b2b215731398c"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"ccef396a27704cc1b3fbd5a073778bf2c231d311","unresolved":false,"context_lines":[{"line_number":84,"context_line":"---------------------"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Extra boolean option in nova.conf - cleanup_shelved_instances. Defaults to"},{"line_number":87,"context_line":"False to keep current behaviour."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Performance Impact"},{"line_number":90,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa6399be_dfb05a8e","line":87,"updated":"2016-10-12 00:07:07.000000000","message":"I\u0027m not sure why we\u0027d have a flag? This is just a straight bug fix right? Why would you want the old behaviour?","commit_id":"8848f73485463d89ca5531cd5b6b2b215731398c"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"ebec54f0fd709a255746b38f75801343546b24a6","unresolved":false,"context_lines":[{"line_number":84,"context_line":"---------------------"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Extra boolean option in nova.conf - cleanup_shelved_instances. Defaults to"},{"line_number":87,"context_line":"False to keep current behaviour."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Performance Impact"},{"line_number":90,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa6399be_1a91009a","line":87,"in_reply_to":"fa6399be_dfb05a8e","updated":"2016-10-12 00:27:57.000000000","message":"If you look at it as a bug fix, I guess you\u0027re right. I\u0027ve removed the flag","commit_id":"8848f73485463d89ca5531cd5b6b2b215731398c"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"94fa5f0b7a9e41022b45570eeffa7f26ffb8af99","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/cleanup-running-shelved-instances"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Allow cleaning erroneously active instances when they are shelved or"},{"line_number":14,"context_line":"shelved-offloaded in the DB"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_3aca62a2","line":13,"range":{"start_line":13,"start_character":58,"end_line":13,"end_character":65},"updated":"2016-10-21 12:10:06.000000000","message":"If they are only shelved, and not shelved-offloaded, they are meant to stay on the hypervisor and not be deleted. For sure, they shouldn\u0027t be running.","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"That can cause serious problems for the user - i.e. instance that if marked as"},{"line_number":26,"context_line":"shelved-offloaded, but actually is running, and is later started by the user,"},{"line_number":27,"context_line":"will cause the same instance to run twice in a cluster, and 2 instances can"},{"line_number":28,"context_line":"write to the same volume and cause data corruption."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_93e4543c","line":28,"range":{"start_line":27,"start_character":32,"end_line":28,"end_character":51},"updated":"2016-10-20 21:27:32.000000000","message":"What virt driver were you using btw? libvirt, vmware, other?","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"That can cause serious problems for the user - i.e. instance that if marked as"},{"line_number":26,"context_line":"shelved-offloaded, but actually is running, and is later started by the user,"},{"line_number":27,"context_line":"will cause the same instance to run twice in a cluster, and 2 instances can"},{"line_number":28,"context_line":"write to the same volume and cause data corruption."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_86eea689","line":28,"range":{"start_line":27,"start_character":32,"end_line":28,"end_character":51},"in_reply_to":"da6895a0_93e4543c","updated":"2016-10-22 00:27:28.000000000","message":"libvirt","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Actually-hapened to me - The user has a shelved-offloaded instance. He has"},{"line_number":34,"context_line":"requested to unshelve the instance. nova-compute tries to spawn the instance"},{"line_number":35,"context_line":"in the hypervisor. The hypervisor is really busy and slow, and reports a"},{"line_number":36,"context_line":"timeout. nova-compute tries to clean the instance, but it didn\u0027t spawn yet, it"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_d3edec26","line":33,"range":{"start_line":33,"start_character":9,"end_line":33,"end_character":16},"updated":"2016-10-20 21:27:32.000000000","message":"happened","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Actually-hapened to me - The user has a shelved-offloaded instance. He has"},{"line_number":34,"context_line":"requested to unshelve the instance. nova-compute tries to spawn the instance"},{"line_number":35,"context_line":"in the hypervisor. The hypervisor is really busy and slow, and reports a"},{"line_number":36,"context_line":"timeout. nova-compute tries to clean the instance, but it didn\u0027t spawn yet, it"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_66a8b220","line":33,"range":{"start_line":33,"start_character":9,"end_line":33,"end_character":16},"in_reply_to":"da6895a0_d3edec26","updated":"2016-10-22 00:27:28.000000000","message":"Done","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Actually-hapened to me - The user has a shelved-offloaded instance. He has"},{"line_number":34,"context_line":"requested to unshelve the instance. nova-compute tries to spawn the instance"},{"line_number":35,"context_line":"in the hypervisor. The hypervisor is really busy and slow, and reports a"},{"line_number":36,"context_line":"timeout. nova-compute tries to clean the instance, but it didn\u0027t spawn yet, it"},{"line_number":37,"context_line":"will spawn in a few seconds, and the unshelving fails - the instance goes back"},{"line_number":38,"context_line":"to being shelved-offloaded."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_334b882d","line":36,"range":{"start_line":35,"start_character":19,"end_line":36,"end_character":8},"updated":"2016-10-20 21:27:32.000000000","message":"Which hypervisor? Do you have details on this timeout and the stacktrace when unshelve fails in a bug report?","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Actually-hapened to me - The user has a shelved-offloaded instance. He has"},{"line_number":34,"context_line":"requested to unshelve the instance. nova-compute tries to spawn the instance"},{"line_number":35,"context_line":"in the hypervisor. The hypervisor is really busy and slow, and reports a"},{"line_number":36,"context_line":"timeout. nova-compute tries to clean the instance, but it didn\u0027t spawn yet, it"},{"line_number":37,"context_line":"will spawn in a few seconds, and the unshelving fails - the instance goes back"},{"line_number":38,"context_line":"to being shelved-offloaded."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_26007a34","line":36,"range":{"start_line":35,"start_character":19,"end_line":36,"end_character":8},"in_reply_to":"da6895a0_334b882d","updated":"2016-10-22 00:27:28.000000000","message":"I have it, I will put it in the bug report","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":38,"context_line":"to being shelved-offloaded."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"The user then sees it, and re-tries the unshelving. The scheduler returns a"},{"line_number":41,"context_line":"different node, and the spawning is successful. We now have 2 instances of the"},{"line_number":42,"context_line":"same VM, on different nodes, writing to the same volume, and a data-corruption"},{"line_number":43,"context_line":"occurs."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_9369b48e","line":43,"range":{"start_line":41,"start_character":48,"end_line":43,"end_character":7},"updated":"2016-10-20 21:27:32.000000000","message":"Above you said the hypervisor timed out trying to spawn the instance, but it did actually spawn it? That seems like a bug in the hypervisor, not nova.","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":38,"context_line":"to being shelved-offloaded."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"The user then sees it, and re-tries the unshelving. The scheduler returns a"},{"line_number":41,"context_line":"different node, and the spawning is successful. We now have 2 instances of the"},{"line_number":42,"context_line":"same VM, on different nodes, writing to the same volume, and a data-corruption"},{"line_number":43,"context_line":"occurs."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_46fd6e39","line":43,"range":{"start_line":41,"start_character":48,"end_line":43,"end_character":7},"in_reply_to":"da6895a0_9369b48e","updated":"2016-10-22 00:27:28.000000000","message":"You\u0027re right, it is a hypervisor bug, but you still want nova to handle it, and not cause a preventable data-corruption","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"nova-compute already have a mechanism of cleanup of running-deleted instances,"},{"line_number":50,"context_line":"controlled by the running_deleted_instance_action configuration parameter."},{"line_number":51,"context_line":"This suggests to make the same mechanism cleaning-up shelved-offloaeded"},{"line_number":52,"context_line":"instances on the same loop."},{"line_number":53,"context_line":""},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_f38110fd","line":52,"range":{"start_line":51,"start_character":0,"end_line":52,"end_character":27},"updated":"2016-10-20 21:27:32.000000000","message":"So the problem with this is, _cleanup_running_deleted_instances calls _running_deleted_instances which gets the instances from the database marked as delete on \u0027this host\u0027. When you have an instance in shelved-offloaded state, the instance.host field is None - because the instance in the db is just a placeholder for when you unshelve it.\n\nSo this doesn\u0027t really work the same. You could run this periodic task from the compute manager, but if you can\u0027t filter on host then the compute managers are all going to be getting all instances on all hosts in this state - and you can\u0027t check the hypervisor if they are running that guest because the instance list isn\u0027t filtered by host.\n\nSo I\u0027m not really sure how you expect this to work.","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"nova-compute already have a mechanism of cleanup of running-deleted instances,"},{"line_number":50,"context_line":"controlled by the running_deleted_instance_action configuration parameter."},{"line_number":51,"context_line":"This suggests to make the same mechanism cleaning-up shelved-offloaeded"},{"line_number":52,"context_line":"instances on the same loop."},{"line_number":53,"context_line":""},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_068a96bf","line":52,"range":{"start_line":51,"start_character":0,"end_line":52,"end_character":27},"in_reply_to":"da6895a0_f38110fd","updated":"2016-10-22 00:27:28.000000000","message":"nova-compute can check which instances are currently running in the hypervisor, and check in the DB that they are not shelved. So it can filter on the VM ids or instance ids","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":55,"context_line":"Alternatives"},{"line_number":56,"context_line":"------------"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"None"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Data model impact"},{"line_number":61,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_b38eb825","line":58,"updated":"2016-10-20 21:27:32.000000000","message":"Sure there are, manually cleanup the guest that shouldn\u0027t be running on the hypervisor after opening a support ticket to the cloud operator.","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":55,"context_line":"Alternatives"},{"line_number":56,"context_line":"------------"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"None"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Data model impact"},{"line_number":61,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_86bce66d","line":58,"in_reply_to":"da6895a0_b38eb825","updated":"2016-10-22 00:27:28.000000000","message":"And how can the operator clean the instances? Only choice is to ssh to the node, and to kill the hypervisor\u0027s instances. Is that a valid alternative?","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0532740f55478fc30f70436997f9cc4b4b64410b","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Notifications impact"},{"line_number":76,"context_line":"--------------------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"No change to notifications, but the loop that reaps instances, the calls"},{"line_number":79,"context_line":"shutdown upon instances and the shutdown sends a notification, will do so to"},{"line_number":80,"context_line":"shelved-instances as well as deleted"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Other end user impact"},{"line_number":83,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_f3af507b","line":80,"range":{"start_line":78,"start_character":0,"end_line":80,"end_character":36},"updated":"2016-10-20 21:27:32.000000000","message":"Not really, _cleanup_running_deleted_instances calls _shutdown_instance with notify\u003dFalse so there is no notification for that periodic task when shutting down and deleting the instance.","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"},{"author":{"_account_id":20354,"name":"Shoham Peller","email":"shohamp@gmail.com","username":"shoham.peller"},"change_message_id":"a4533969474bb191cfd7d7cb9c1d6219cb2aa1b1","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Notifications impact"},{"line_number":76,"context_line":"--------------------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"No change to notifications, but the loop that reaps instances, the calls"},{"line_number":79,"context_line":"shutdown upon instances and the shutdown sends a notification, will do so to"},{"line_number":80,"context_line":"shelved-instances as well as deleted"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Other end user impact"},{"line_number":83,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da6895a0_46abae1d","line":80,"range":{"start_line":78,"start_character":0,"end_line":80,"end_character":36},"in_reply_to":"da6895a0_f3af507b","updated":"2016-10-22 00:27:28.000000000","message":"Done","commit_id":"ba3b8209e42a1caba2a955986771d9241de29a1a"}]}