)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Instances auto evacuation"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Unified workload auto-evacuation mechanism for Openstack."},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Change-Id: Ia61ffaa6785ee2c784da8a5573aa04623d231611"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"5a710552_e621924e","line":9,"updated":"2015-12-16 12:57:11.000000000","message":"If you want to be following the rules it\u0027s \"OpenStack\", but that\u0027s a nit.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"}],"specs/instance-auto-evacuation.rst":[{"author":{"_account_id":12404,"name":"Rico Lin","email":"ricolin@ricolky.com","username":"rico.lin"},"change_message_id":"d2c785837fe8e9f11e966a30730f3e8e0ce6e281","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Problem description"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation is essential cloud platforms feature, which allows to"},{"line_number":12,"context_line":"reduce instance downtime. On relatively big environments it could be hard to"},{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_dfacd5a0","line":12,"range":{"start_line":11,"start_character":60,"end_line":12,"end_character":24},"updated":"2016-02-02 03:07:55.000000000","message":"Evacuation is more like a DR solution, even you auto-evacuate to other available host, that instance still not running. Maybe use Live Migrate in this case is more appropriated, since you\u0027re looking for no downtime design.\nOn the other hand we can put heat into this idea as well, like a resource doing all above task.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"6f3d3d1a28dfda49304d95ce381b2f6f2f6afe59","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Problem description"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation is essential cloud platforms feature, which allows to"},{"line_number":12,"context_line":"reduce instance downtime. On relatively big environments it could be hard to"},{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a5ae5dd_6abc0fbb","line":12,"range":{"start_line":11,"start_character":60,"end_line":12,"end_character":24},"in_reply_to":"7a5de9d1_dfacd5a0","updated":"2016-02-08 13:48:23.000000000","message":"I believe this is about an evacuation with downtime aka \"resurrection\", which is visible to the instance as a reboot. Live migrate has nothing to the evacuation, which is just a confusing term. So, it does reduce downtime by the meaning that limited downtime is better then unlimited one.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":17211,"name":"Brandon Palm","email":"bapalm@us.ibm.com","username":"bapalm"},"change_message_id":"105df6fd7a89a6c0aff08db6cc3818629f88f5b0","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation is essential cloud platforms feature, which allows to"},{"line_number":12,"context_line":"reduce instance downtime. On relatively big environments it could be hard to"},{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "},{"line_number":16,"context_line":"workload migration manually, Openstack needs a unified approach"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_ce11655f","line":13,"range":{"start_line":13,"start_character":67,"end_line":13,"end_character":76},"updated":"2016-01-27 16:07:36.000000000","message":"OpenStack*","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"ef9c6c2b6af1df553176c0a9a313e290e2109fc9","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation is essential cloud platforms feature, which allows to"},{"line_number":12,"context_line":"reduce instance downtime. On relatively big environments it could be hard to"},{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "},{"line_number":16,"context_line":"workload migration manually, Openstack needs a unified approach"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a5ae5dd_4b004b66","line":13,"range":{"start_line":13,"start_character":38,"end_line":13,"end_character":46},"updated":"2016-02-08 06:26:38.000000000","message":"workloads*","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":12404,"name":"Rico Lin","email":"ricolin@ricolky.com","username":"rico.lin"},"change_message_id":"d2c785837fe8e9f11e966a30730f3e8e0ce6e281","unresolved":false,"context_lines":[{"line_number":12,"context_line":"reduce instance downtime. On relatively big environments it could be hard to"},{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "},{"line_number":16,"context_line":"workload migration manually, Openstack needs a unified approach"},{"line_number":17,"context_line":"for workload auto-evacuation. Proposed functionality should enable automatic"},{"line_number":18,"context_line":"detection of potential problems and evacuate instances from suffering node.  "}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_b9a1f995","line":15,"range":{"start_line":15,"start_character":64,"end_line":15,"end_character":65},"updated":"2016-02-02 03:07:55.000000000","message":"extra white space","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":17211,"name":"Brandon Palm","email":"bapalm@us.ibm.com","username":"bapalm"},"change_message_id":"105df6fd7a89a6c0aff08db6cc3818629f88f5b0","unresolved":false,"context_lines":[{"line_number":13,"context_line":"track all potential problems and make workoads evacuation in time. Openstack"},{"line_number":14,"context_line":"project already has instruments for gathering statistics, migration"},{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "},{"line_number":16,"context_line":"workload migration manually, Openstack needs a unified approach"},{"line_number":17,"context_line":"for workload auto-evacuation. Proposed functionality should enable automatic"},{"line_number":18,"context_line":"detection of potential problems and evacuate instances from suffering node.  "},{"line_number":19,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_8e0ccd43","line":16,"range":{"start_line":16,"start_character":29,"end_line":16,"end_character":38},"updated":"2016-01-27 16:07:36.000000000","message":"OpenStack*","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":12404,"name":"Rico Lin","email":"ricolin@ricolky.com","username":"rico.lin"},"change_message_id":"d2c785837fe8e9f11e966a30730f3e8e0ce6e281","unresolved":false,"context_lines":[{"line_number":15,"context_line":"workloads, and workflow orchestration. While it possible to make "},{"line_number":16,"context_line":"workload migration manually, Openstack needs a unified approach"},{"line_number":17,"context_line":"for workload auto-evacuation. Proposed functionality should enable automatic"},{"line_number":18,"context_line":"detection of potential problems and evacuate instances from suffering node.  "},{"line_number":19,"context_line":""},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_59ac858b","line":18,"range":{"start_line":18,"start_character":75,"end_line":18,"end_character":77},"updated":"2016-02-02 03:07:55.000000000","message":"extra white space","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":21,"context_line":"Proposed change"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Process of workload evacuation could be splitted in 3 parts:"},{"line_number":25,"context_line":" - gathering statistics"},{"line_number":26,"context_line":" - triggering auto-evacuation"},{"line_number":27,"context_line":" - moving workloads"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a710552_061dbe0e","line":24,"updated":"2015-12-16 12:57:11.000000000","message":"s/splitted in/split into/","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":43,"context_line":"|                              +--------+   |"},{"line_number":44,"context_line":"|                                           |"},{"line_number":45,"context_line":"+-------------------------------------------+"},{"line_number":46,"context_line":"Ceilometer has alarms abstraction that allows to specify"},{"line_number":47,"context_line":"metrics to watch, threshold value, and uri that should be"},{"line_number":48,"context_line":"requested. When the metric meets threshold ceilometer notifies"},{"line_number":49,"context_line":"evacuation workflow controller that triggers the whole process."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a710552_a61b8a1a","line":46,"updated":"2015-12-16 12:57:11.000000000","message":"As of Mitaka two things have happened that could be clarified here: 1) The project formerly known as Ceilometer is now calling itself Telemetry and covers several code repositories. One of those is called ceilometer and covers the meter and event gathering (via polling and notifications) and api on existing storage models. 2) The code for alarming has been moved to \u0027aodh\u0027 (pronounced like the letter \u0027a\u0027 in \u0027say\u0027 or \u0027day\u0027).\n\nI mention because it has some impact on where some of the activity being handled by the proposed workflow would happen:\n\nCeilometer pollsters and notification agents would \"hear\" measures from polling nova-compute or receiving notifications from Nova. Aodh would have threshold alarms that check these thresholds at various intervals.\n\nA few parties have pointed out that the sheer volume of data required for the measure-\u003estore\u003c-query model described above limits the scalability and responsiveness of that solution. An alternative is in progress which uses an event evaluator (see blog posting: https://01.org/openstack/blogs/cabuerger/2015/aodh-alarm-events-openstack ).\n\nWhatever the situation, the eventual outcome of an alarm is a webhook.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Auto-evacuation controller makes decision, which type of instance migration"},{"line_number":52,"context_line":"is more preferable. Decision should be made based on info, that ceilometer"},{"line_number":53,"context_line":"had provided."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":56,"context_line":"-live-migration"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a710552_c616d6f0","line":53,"updated":"2015-12-16 12:57:11.000000000","message":"This paragraph is probably the most important part of the spec, but is too short. What is the information required to make a decision? Does that information need to be an observation over a window of time or can it operate from a single instance? What are the different scenarios that lead to the different types of migration?","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"ef9c6c2b6af1df553176c0a9a313e290e2109fc9","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Auto-evacuation controller makes decision, which type of instance migration"},{"line_number":52,"context_line":"is more preferable. Decision should be made based on info, that ceilometer"},{"line_number":53,"context_line":"had provided."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":56,"context_line":"-live-migration"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a5ae5dd_ebcd7766","line":53,"in_reply_to":"5a710552_c616d6f0","updated":"2016-02-08 06:26:38.000000000","message":"Also it is worth to mention how to treat information ceilometer has provide. What does it mean that compute service is down? Does it mean that whole compute host is down, or just the compute service? And what if the compute service is down, but libvirt is perfectly fine and so the VMs? I think those are the questions, which should be answered first.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":8215,"name":"Tim Hinrichs","email":"timothy.l.hinrichs@gmail.com","username":"thinrich"},"change_message_id":"271cfa051fe2b8db6659c4bd218a84883d9cb03c","unresolved":false,"context_lines":[{"line_number":56,"context_line":"-live-migration"},{"line_number":57,"context_line":"-cold-migration"},{"line_number":58,"context_line":"-evacuation(rebuild)"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Alternatives"},{"line_number":61,"context_line":"------------"},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_761aaec7","line":59,"updated":"2016-02-02 01:51:29.000000000","message":"I might recommend trying this in Congress.  It lets you write conditions under which scripts should execute.  Those conditions can utilize information from many different projects (such as nova and ceilometer).  The script can be a simple wrapper that starts a Mistral workflow.  You can even compute parameters to the script based on information from many different projects.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"e14477424625756c812e259503ad7e7a4b66a02a","unresolved":false,"context_lines":[{"line_number":56,"context_line":"-live-migration"},{"line_number":57,"context_line":"-cold-migration"},{"line_number":58,"context_line":"-evacuation(rebuild)"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Alternatives"},{"line_number":61,"context_line":"------------"},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aa78d24_19a6c1c0","line":59,"in_reply_to":"7a5de9d1_761aaec7","updated":"2016-07-13 02:45:54.000000000","message":"(resurrecting an old comment because I\u0027m late to the party)\n\nAgreed, we discussed involving Congress in Austin and I think there is a lot of merit to a Ceilometer -\u003e Monasca -\u003e Congress -\u003e Mistral event flow and that creating the glue for each stage should be part of this spec.\n\nHaving said that, also I think we need to recognise that this may not be a one-size-fits-all situation and to allow for some flexibility in who/how the mistral workflow is initiated.  Some distributors may wish to trigger Mistral directly from Ceilometer, others may use a completely different method of detecting compute and instance death.\n\nWhich is a long way of saying, the Mistral workflows are the most important part of this IMHO :-)","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":12404,"name":"Rico Lin","email":"ricolin@ricolky.com","username":"rico.lin"},"change_message_id":"d2c785837fe8e9f11e966a30730f3e8e0ce6e281","unresolved":false,"context_lines":[{"line_number":61,"context_line":"------------"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"-Not to introduce this functionality"},{"line_number":64,"context_line":"-Use Heat instead of Mistral"},{"line_number":65,"context_line":"-Create API in nova for host auto-evacuation - nova scheduler could pick appropriate host"},{"line_number":66,"context_line":"for instance being migrated, so nova could acts as workflow controller. Ceilometer will"},{"line_number":67,"context_line":"execute nova host-evacuate then."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a5de9d1_f90d318f","line":64,"range":{"start_line":64,"start_character":5,"end_line":64,"end_character":28},"updated":"2016-02-02 03:07:55.000000000","message":"no need for capital","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":63,"context_line":"-Not to introduce this functionality"},{"line_number":64,"context_line":"-Use Heat instead of Mistral"},{"line_number":65,"context_line":"-Create API in nova for host auto-evacuation - nova scheduler could pick appropriate host"},{"line_number":66,"context_line":"for instance being migrated, so nova could acts as workflow controller. Ceilometer will"},{"line_number":67,"context_line":"execute nova host-evacuate then."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a710552_66d4c21f","line":66,"updated":"2015-12-16 12:57:11.000000000","message":"s/acts/act/","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"0a90ed540053f1d938849d8c80990202e50a3ae5","unresolved":false,"context_lines":[{"line_number":64,"context_line":"-Use Heat instead of Mistral"},{"line_number":65,"context_line":"-Create API in nova for host auto-evacuation - nova scheduler could pick appropriate host"},{"line_number":66,"context_line":"for instance being migrated, so nova could acts as workflow controller. Ceilometer will"},{"line_number":67,"context_line":"execute nova host-evacuate then."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"Implementation"},{"line_number":70,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a710552_86d76e22","line":67,"updated":"2015-12-16 12:57:11.000000000","message":"It\u0027s probably best to be more clear about what ceilometer/telemetry/aodh is doing here. It\u0027s not executing nova itself. All it can do is cause a webhook URL to receive a POST.","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"ef9c6c2b6af1df553176c0a9a313e290e2109fc9","unresolved":false,"context_lines":[{"line_number":65,"context_line":"-Create API in nova for host auto-evacuation - nova scheduler could pick appropriate host"},{"line_number":66,"context_line":"for instance being migrated, so nova could acts as workflow controller. Ceilometer will"},{"line_number":67,"context_line":"execute nova host-evacuate then."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"Implementation"},{"line_number":70,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":71,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a5ae5dd_8be3a3d8","line":68,"updated":"2016-02-08 06:26:38.000000000","message":"There is couple of other alternatives for auto evacuation, few of them are already in production. like Masakari for example. Maybe it is worth to mention it here.\n\nhttps://etherpad.openstack.org/p/automatic-evacuation","commit_id":"5390757ecdf628c5f82c976d376a22b92a7a7071"},{"author":{"_account_id":15397,"name":"Toshikazu Kazu Ichikawa","email":"t.ichikawa@ntt.com","username":"kazu"},"change_message_id":"2c8f07a1e8b5d78ab468c844414b56d45df8fb71","unresolved":false,"context_lines":[{"line_number":20,"context_line":"Proposed change"},{"line_number":21,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":24,"context_line":" - gathering statistics"},{"line_number":25,"context_line":" - triggering auto-evacuation"},{"line_number":26,"context_line":" - moving workloads"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_f61634a6","line":23,"range":{"start_line":23,"start_character":0,"end_line":23,"end_character":59},"updated":"2016-03-15 16:15:06.000000000","message":"I believe we also need to add \"fencing or locking to instance store access,\" as you mentioned at Alternatives section. Otherwise, it may cause data corruption by having two VM processes accessing same instance store at the same time in worst case.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"eb48c2dc9bd6e73d69703e480797e0aafd70546f","unresolved":false,"context_lines":[{"line_number":20,"context_line":"Proposed change"},{"line_number":21,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":24,"context_line":" - gathering statistics"},{"line_number":25,"context_line":" - triggering auto-evacuation"},{"line_number":26,"context_line":" - moving workloads"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_a2ab811c","line":23,"range":{"start_line":23,"start_character":0,"end_line":23,"end_character":59},"in_reply_to":"1af94dfe_f61634a6","updated":"2016-03-16 19:16:10.000000000","message":"Yes, fencing is an absolutely critical requirement, either at the node level or the resource level.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"83318d18ca5016ba6decb8e58871b393b2f59574","unresolved":false,"context_lines":[{"line_number":21,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":24,"context_line":" - gathering statistics"},{"line_number":25,"context_line":" - triggering auto-evacuation"},{"line_number":26,"context_line":" - moving workloads"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_2ea0339f","line":24,"updated":"2016-03-15 21:34:52.000000000","message":"Once detect the failure, we could use Instance lock/unlock to lock the instance for users, till we finished the auto-evacuation.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"26f0388a3f7772b97e89d2a44a6f0ba86effd7b7","unresolved":false,"context_lines":[{"line_number":21,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":24,"context_line":" - gathering statistics"},{"line_number":25,"context_line":" - triggering auto-evacuation"},{"line_number":26,"context_line":" - moving workloads"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_97306e58","line":24,"in_reply_to":"1af94dfe_2ea0339f","updated":"2016-03-16 06:20:29.000000000","message":"Could you elaborate a bit more on that? What do you mean by locking instance? If we detect the host failure (and fence it) there is nothing anyone could do on such situation, unless the rebuilding is completed. What the benefit is for locking the instance in such situation?","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"d0fd41c55e18993d34c32de4872581a1a1f1d6b6","unresolved":false,"context_lines":[{"line_number":21,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":24,"context_line":" - gathering statistics"},{"line_number":25,"context_line":" - triggering auto-evacuation"},{"line_number":26,"context_line":" - moving workloads"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa0719c6_82268c14","line":24,"in_reply_to":"1af94dfe_97306e58","updated":"2016-03-21 11:08:31.000000000","message":"IIUC, this is about DLM in tooz (https://specs.openstack.org/openstack/openstack-specs/specs/chronicles-of-a-dlm.html)? And that is supposed to be an alternative to the fencing.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"67d882733e3a2ea2efd6e3dcce99ba1d53987e89","unresolved":false,"context_lines":[{"line_number":56,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":57,"context_line":" - live-migration"},{"line_number":58,"context_line":" - cold-migration"},{"line_number":59,"context_line":" - evacuation aka resurrection (rebuild)"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Alternatives"},{"line_number":62,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_1da478a4","line":59,"updated":"2016-03-16 09:15:59.000000000","message":"This might be a useful reference here on why we have so many options. It all depends on the technology (in particular storage) being used by the instance, and the specific failure mode/use case:\nhttp://developer.openstack.org/api-guide/compute/server_concepts.html","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"67d882733e3a2ea2efd6e3dcce99ba1d53987e89","unresolved":false,"context_lines":[{"line_number":71,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":72,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":73,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Implementation"},{"line_number":76,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_dd0cd096","line":74,"updated":"2016-03-16 09:15:59.000000000","message":"There has been talk about en users specifying a preference on what happens to the VM on failure, which could be exposed via the Nova API.\n\nLike delete on failure, or reboot on failure, etc. Not really got a handle on the options needed there yet, but this work might inform that.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"eb48c2dc9bd6e73d69703e480797e0aafd70546f","unresolved":false,"context_lines":[{"line_number":71,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":72,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":73,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Implementation"},{"line_number":76,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_58738a80","line":74,"in_reply_to":"1af94dfe_dd0cd096","updated":"2016-03-16 19:16:10.000000000","message":"That\u0027s correct.  Ultimately I suspect that this needs to be handled by Congress as per-cloud policy.  Actually even per-cloud wouldn\u0027t necessarily be fine-grained enough; I expect many clouds would want different HA policies per AZ, per project, and even per instance.  Here\u0027s a nice fragment of a session from Tokyo which touched on this:\n\nhttps://youtu.be/nz4kEZcmxr4?t\u003d8m47s","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"eb48c2dc9bd6e73d69703e480797e0aafd70546f","unresolved":false,"context_lines":[{"line_number":78,"context_line":"Assignee(s)"},{"line_number":79,"context_line":"-----------"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"* Adam Spiers \u003caspiers@suse.com\u003e, aspiers"},{"line_number":82,"context_line":"* Dawid Deja \u003cdawid.deja@intel.com\u003e, ddeja"},{"line_number":83,"context_line":"* Masahiro Ikeda \u003cikedamsh@nttdata.co.jp\u003e, masahito"},{"line_number":84,"context_line":"* Andrew Beekhof \u003candrew@beekhof.net\u003e, beekhof"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_b3663127","line":81,"updated":"2016-03-16 19:16:10.000000000","message":"I was slightly surprised to find myself as an assignee in this spec, but that\u0027s OK :-)","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":8878,"name":"Masahito Muroi","email":"masahito.muroi@g.softbank.co.jp","username":"masa"},"change_message_id":"fcf1852daf90c9674710c3071cfc8c135641ab79","unresolved":false,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"* Adam Spiers \u003caspiers@suse.com\u003e, aspiers"},{"line_number":82,"context_line":"* Dawid Deja \u003cdawid.deja@intel.com\u003e, ddeja"},{"line_number":83,"context_line":"* Masahiro Ikeda \u003cikedamsh@nttdata.co.jp\u003e, masahito"},{"line_number":84,"context_line":"* Andrew Beekhof \u003candrew@beekhof.net\u003e, beekhof"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Work Items"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3afc51ec_97ee9eb2","line":83,"range":{"start_line":83,"start_character":2,"end_line":83,"end_character":51},"updated":"2016-03-14 03:13:37.000000000","message":"I think you\u0027ve mistaken me for him.  If you want to listing someone who attend HA meeting, it would be me.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"eb48c2dc9bd6e73d69703e480797e0aafd70546f","unresolved":false,"context_lines":[{"line_number":87,"context_line":"----------"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"* Improve fence compute agent to ensure STONITH of a compute node before to"},{"line_number":90,"context_line":"  initiate instances resurrection (evacuation). INPROGRESS: beekhof, aspiers."},{"line_number":91,"context_line":"* PoC Mistral workflow to evacuate instances [#]_. DONE: ddeja."},{"line_number":92,"context_line":"* PoC Pacemaker fence agent to initiate Mistral workflows. TBD: ddeja"},{"line_number":93,"context_line":"* PoC Masakari-based fence agent to initiate Mistral workflows .TBD: masahito"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1af94dfe_5736b8e6","line":90,"updated":"2016-03-16 19:16:10.000000000","message":"This is already done by both the Red Hat and SUSE implementations, which only trigger the fence_compute fencing agent after the normal STONITH device has been triggered on the node.  I suspect the same is true with masakari, but I\u0027m not 100% sure.","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":8878,"name":"Masahito Muroi","email":"masahito.muroi@g.softbank.co.jp","username":"masa"},"change_message_id":"fcf1852daf90c9674710c3071cfc8c135641ab79","unresolved":false,"context_lines":[{"line_number":122,"context_line":".. [#] `Masakari \u003chttps://github.com/ntt-sic/masakari\u003e`_."},{"line_number":123,"context_line":".. [#] `Instances HA using Mistral \u003chttps://github.com/gryf/mistral-evacuate\u003e`_."},{"line_number":124,"context_line":".. [#] `Automatic evacuation etherpad \u003chttps://etherpad.openstack.org/p/automatic-evacuation\u003e`_."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":".. note::"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"  This work is licensed under a Creative Commons Attribution 3.0 Unported License."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3afc51ec_57d7f6be","line":125,"updated":"2016-03-14 03:13:37.000000000","message":"Do you know the UserStory about High Availability for Virtual Machine[1]? IMO, it would be better to mention about it in the spec and collaborate with the team.\n\n1. http://specs.openstack.org/openstack/openstack-user-stories/user-stories/draft/ha_vm.html","commit_id":"d441158738367b93cec382fc68f5da8d5fd4d97c"},{"author":{"_account_id":19206,"name":"Dawid Deja","email":"dawdej@gmail.com","username":"ddeja"},"change_message_id":"4dc629e078c7d1e6687783d1e35ecf57d2f1846d","unresolved":false,"context_lines":[{"line_number":110,"context_line":"* Improve fence compute agent to ensure STONITH of a compute node before to"},{"line_number":111,"context_line":"  initiate instances resurrection (evacuation). DONE: beekhof, aspiers."},{"line_number":112,"context_line":"* PoC Mistral workflow to evacuate instances [#]_. DONE: ddeja."},{"line_number":113,"context_line":"* PoC Pacemaker fence agent to initiate Mistral workflows. TBD: ddeja"},{"line_number":114,"context_line":"* PoC Masakari-based fence agent to initiate Mistral workflows .TBD: masahito"},{"line_number":115,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":116,"context_line":"  running workflows. By results, use either Masakari or Mistral or"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa0719c6_6fe7e0e4","line":113,"range":{"start_line":113,"start_character":2,"end_line":113,"end_character":69},"updated":"2016-03-21 11:23:14.000000000","message":"This is done, only needs some how-to\nhttps://github.com/gryf/mistral-evacuate/blob/master/fence_evacuate.py","commit_id":"d11de2fe613876ae79d198497c525725c75e211d"},{"author":{"_account_id":19206,"name":"Dawid Deja","email":"dawdej@gmail.com","username":"ddeja"},"change_message_id":"4dc629e078c7d1e6687783d1e35ecf57d2f1846d","unresolved":false,"context_lines":[{"line_number":119,"context_line":"Dependencies"},{"line_number":120,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"- `Mistral HA \u003chttps://blueprints.launchpad.net/mistral/+spec/mistral-ha-spec\u003e`_."},{"line_number":123,"context_line":"- `DLM (optional to fencing)"},{"line_number":124,"context_line":"  \u003chttps://specs.openstack.org/openstack/openstack-specs/specs/chronicles-of-a-dlm.html`\u003e."},{"line_number":125,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa0719c6_0f3104a3","line":122,"updated":"2016-03-21 11:23:14.000000000","message":"This was resolved in https://bugs.launchpad.net/mistral/+bug/1502120 but still has some limitations. Nevertheless, I think mentioned fix should be added here (maybe as a reference?)","commit_id":"d11de2fe613876ae79d198497c525725c75e211d"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"066849f9f974f6d306906e1683566f8daaf4de6b","unresolved":false,"context_lines":[{"line_number":27,"context_line":"  cross-projects consensus. Low level and implementation specific things shall"},{"line_number":28,"context_line":"  go into the projects\u0027 specs."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Process of workload evacuation could be split into 3 parts:"},{"line_number":31,"context_line":" - gathering statistics"},{"line_number":32,"context_line":" - fencing (a node/resource level) or locking to instance store access"},{"line_number":33,"context_line":" - triggering auto-evacuation"}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa0719c6_cc3d01bd","line":30,"updated":"2016-03-22 22:07:02.000000000","message":"This is now 4 parts, not 3.","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"066849f9f974f6d306906e1683566f8daaf4de6b","unresolved":false,"context_lines":[{"line_number":77,"context_line":"* Let users to make a preference on what happens to the VM on failure, which"},{"line_number":78,"context_line":"  could be exposed via the Nova API, like delete on failure, or reboot on"},{"line_number":79,"context_line":"  failure, or resurrect (auto-evac), or a suchlike."},{"line_number":80,"context_line":"* Use Heat instead of Mistral."},{"line_number":81,"context_line":"* Create API in nova for host auto-evacuation - nova scheduler could pick"},{"line_number":82,"context_line":"  appropriate host for instance being migrated, so nova could acts as workflow"},{"line_number":83,"context_line":"  controller. Ceilometer will execute nova host-evacuate then."}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa0719c6_0ce41939","line":80,"updated":"2016-03-22 22:07:02.000000000","message":"Is Heat already capable of executing arbitrary workflows, like evacuation?  That would be news to me, although my Heat knowledge is outdated.","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"d71188455dde277d37c97b6e131ac68d187dda90","unresolved":false,"context_lines":[{"line_number":77,"context_line":"* Let users to make a preference on what happens to the VM on failure, which"},{"line_number":78,"context_line":"  could be exposed via the Nova API, like delete on failure, or reboot on"},{"line_number":79,"context_line":"  failure, or resurrect (auto-evac), or a suchlike."},{"line_number":80,"context_line":"* Use Heat instead of Mistral."},{"line_number":81,"context_line":"* Create API in nova for host auto-evacuation - nova scheduler could pick"},{"line_number":82,"context_line":"  appropriate host for instance being migrated, so nova could acts as workflow"},{"line_number":83,"context_line":"  controller. Ceilometer will execute nova host-evacuate then."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7a1b2928_0a8f3d24","line":80,"in_reply_to":"fa0719c6_0ce41939","updated":"2016-04-11 10:06:13.000000000","message":"I don\u0027t know. Would be nice if Heat community could confirm or decline that alternative.","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"066849f9f974f6d306906e1683566f8daaf4de6b","unresolved":false,"context_lines":[{"line_number":84,"context_line":"* Use Pacemaker remote [#]_, fence agents [#]_, Masakari [#]_ auto-evacuation"},{"line_number":85,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":86,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":87,"context_line":"* Use DLM from Tooz [#]_ instead of fencing."},{"line_number":88,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa0719c6_acf40df2","line":87,"updated":"2016-03-22 22:07:02.000000000","message":"Please can you explain (in the spec) why a DLM would be a sufficient replacement for fencing?  I can\u0027t imagine how that would work.  There was recent discussion on whether a DLM like tooz needs fencing hooks, but this topic seems still WIP: http://thread.gmane.org/gmane.comp.cloud.openstack.devel/76578/focus\u003d76812","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"d71188455dde277d37c97b6e131ac68d187dda90","unresolved":false,"context_lines":[{"line_number":84,"context_line":"* Use Pacemaker remote [#]_, fence agents [#]_, Masakari [#]_ auto-evacuation"},{"line_number":85,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":86,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":87,"context_line":"* Use DLM from Tooz [#]_ instead of fencing."},{"line_number":88,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7a1b2928_6a235106","line":87,"in_reply_to":"fa0719c6_acf40df2","updated":"2016-04-11 10:06:13.000000000","message":"Technically, DLM may guarantee a critical section will be held exclusively, so kinda of no fencing is required additionally. For example, there is no fencing in zookeeper (I mean STONITH here), instead there are fencing tokens which prevent data from unexpected changes... So that is an alternative to STONITH, in fact. Should I rephrase? And I have no proves if this consistent DLM is doable in practice, but I\u0027m pretty sure ZK folks have ;)","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"066849f9f974f6d306906e1683566f8daaf4de6b","unresolved":false,"context_lines":[{"line_number":111,"context_line":"  initiate instances resurrection (evacuation). DONE: beekhof, aspiers."},{"line_number":112,"context_line":"* PoC Mistral workflow to evacuate instances [#]_. DONE: ddeja."},{"line_number":113,"context_line":"* PoC Pacemaker fence agent [#]_ to initiate Mistral workflows. DONE: ddeja"},{"line_number":114,"context_line":"* PoC Masakari-based fence agent to initiate Mistral workflows. TBD: masahito"},{"line_number":115,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":116,"context_line":"  running workflows. By results, use either Masakari or Mistral or"},{"line_number":117,"context_line":"  a combination of both. For fencing use either Pacemaker with Pacemaker_remote"}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa0719c6_66196c88","line":114,"updated":"2016-03-22 22:07:02.000000000","message":"Maybe we could have an action item to change Masakari to use Pacemaker\u0027s native process-monitoring instead of its own?  I think that would help convergence.","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"f01cdaa0311c90eee804daf9ed2dc33f7da9edbe","unresolved":false,"context_lines":[{"line_number":111,"context_line":"  initiate instances resurrection (evacuation). DONE: beekhof, aspiers."},{"line_number":112,"context_line":"* PoC Mistral workflow to evacuate instances [#]_. DONE: ddeja."},{"line_number":113,"context_line":"* PoC Pacemaker fence agent [#]_ to initiate Mistral workflows. DONE: ddeja"},{"line_number":114,"context_line":"* PoC Masakari-based fence agent to initiate Mistral workflows. TBD: masahito"},{"line_number":115,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":116,"context_line":"  running workflows. By results, use either Masakari or Mistral or"},{"line_number":117,"context_line":"  a combination of both. For fencing use either Pacemaker with Pacemaker_remote"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7a1b2928_2aedd98b","line":114,"in_reply_to":"fa0719c6_66196c88","updated":"2016-04-11 10:07:39.000000000","message":"I\u0027m not sure how should I rephrase that","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"066849f9f974f6d306906e1683566f8daaf4de6b","unresolved":false,"context_lines":[{"line_number":115,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":116,"context_line":"  running workflows. By results, use either Masakari or Mistral or"},{"line_number":117,"context_line":"  a combination of both. For fencing use either Pacemaker with Pacemaker_remote"},{"line_number":118,"context_line":"  or DLM from Tooz for locking."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Dependencies"},{"line_number":121,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa0719c6_cf085330","line":118,"updated":"2016-03-22 22:07:02.000000000","message":"Same comment as above regarding Tooz and fencing.","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"f01cdaa0311c90eee804daf9ed2dc33f7da9edbe","unresolved":false,"context_lines":[{"line_number":115,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":116,"context_line":"  running workflows. By results, use either Masakari or Mistral or"},{"line_number":117,"context_line":"  a combination of both. For fencing use either Pacemaker with Pacemaker_remote"},{"line_number":118,"context_line":"  or DLM from Tooz for locking."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Dependencies"},{"line_number":121,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7a1b2928_6affd134","line":118,"in_reply_to":"fa0719c6_cf085330","updated":"2016-04-11 10:07:39.000000000","message":"answered","commit_id":"fe5c4d0a91008b084dfd4135c1d414e77c8a4f6c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"e53e13a97b7b050bd7a16f8ab8f5711edf2c3579","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  guarantee a critical section to be held exclusively without power management"},{"line_number":91,"context_line":"  or resources isolation related fencing actions to the cluster. For example,"},{"line_number":92,"context_line":"  there is no STONITH in a Zookeeper, instead there are fencing tokens that"},{"line_number":93,"context_line":"  prevent data from unexpected changes."},{"line_number":94,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7a1b2928_af444574","line":93,"updated":"2016-04-11 15:30:58.000000000","message":"That\u0027s fine with Zookeeper, but how would it apply to VMs running via libvirt, with access to shared storage which has no clue about fencing tokens?  In this case, we need to ensure that the VM on the compute node which we *think* is dead (but cannot be sure, because we didn\u0027t fence it) can no longer write to the shared storage.  \n\nIOW, it\u0027s like the diagram here:\n\nhttp://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html#making-the-lock-safe-with-fencing\n\nI can\u0027t imagine how you\u0027d use fencing tokens to ensure that, because it would require every single I/O op from the hypervisor to the storage to go through an extra layer which used fencing tokens, and that sounds extremely complicated, ugly, and likely to have a big impact on I/O performance.  Or am I misunderstanding something?\n\nEither way, if you are suggesting this kind of approach, it would be helpful to include links to relevant pages, e.g.\n\nhttp://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html\nhttp://thread.gmane.org/gmane.comp.cloud.openstack.devel/76578/focus\u003d76825\nhttp://specs.openstack.org/openstack/openstack-specs/specs/chronicles-of-a-dlm.html\n\nAlso, I would suggest that we add a use case to the dlm spec linked immediately above.","commit_id":"335c82d9202cd690519b6c426a899b7dd01481e0"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"538e7b5aad3c645a3996ae7b40533e84f30722eb","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  guarantee a critical section to be held exclusively without power management"},{"line_number":91,"context_line":"  or resources isolation related fencing actions to the cluster. For example,"},{"line_number":92,"context_line":"  there is no STONITH in a Zookeeper, instead there are fencing tokens that"},{"line_number":93,"context_line":"  prevent data from unexpected changes."},{"line_number":94,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a18252c_d27e0969","line":93,"in_reply_to":"7a1b2928_af444574","updated":"2016-04-14 14:48:49.000000000","message":"Ok will adjust. There is a link for the latter spec below","commit_id":"335c82d9202cd690519b6c426a899b7dd01481e0"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Problem description"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation (instances resurrection) is essential cloud platforms feature,"},{"line_number":12,"context_line":"which allows to reduce instance downtime. On relatively big environments it"},{"line_number":13,"context_line":"could be hard to track all potential problems and make workloads evacuation in"},{"line_number":14,"context_line":"time. OpenStack project already has instruments for gathering statistics,"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_8e338961","line":11,"range":{"start_line":11,"start_character":41,"end_line":11,"end_character":53},"updated":"2016-07-13 02:45:14.000000000","message":"is an essential","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation (instances resurrection) is essential cloud platforms feature,"},{"line_number":12,"context_line":"which allows to reduce instance downtime. On relatively big environments it"},{"line_number":13,"context_line":"could be hard to track all potential problems and make workloads evacuation in"},{"line_number":14,"context_line":"time. OpenStack project already has instruments for gathering statistics,"},{"line_number":15,"context_line":"migration workloads, and workflow orchestration [#]_. While it possible to make"},{"line_number":16,"context_line":"workload migration manually, OpenStack needs a unified approach for workload"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_0e4839cb","line":13,"range":{"start_line":13,"start_character":65,"end_line":13,"end_character":75},"updated":"2016-07-13 02:45:14.000000000","message":"evacuate","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":13,"context_line":"could be hard to track all potential problems and make workloads evacuation in"},{"line_number":14,"context_line":"time. OpenStack project already has instruments for gathering statistics,"},{"line_number":15,"context_line":"migration workloads, and workflow orchestration [#]_. While it possible to make"},{"line_number":16,"context_line":"workload migration manually, OpenStack needs a unified approach for workload"},{"line_number":17,"context_line":"auto-evacuation. Proposed functionality should enable automatic detection of"},{"line_number":18,"context_line":"potential problems and evacuate instances from suffering node."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_ce28f1e4","line":16,"range":{"start_line":16,"start_character":0,"end_line":16,"end_character":27},"updated":"2016-07-13 02:45:14.000000000","message":"\"workload migrate manually\" or \"workload migration function manually\"","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  |                                           |"},{"line_number":53,"context_line":"  +-------------------------------------------+"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Ceilometer has alarms abstraction that allows to specify"},{"line_number":56,"context_line":"metrics to watch, threshold value, and uri that should be"},{"line_number":57,"context_line":"requested. When the metric meets threshold ceilometer notifies"},{"line_number":58,"context_line":"evacuation workflow controller that triggers the whole process."}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_aeff6532","line":55,"range":{"start_line":55,"start_character":39,"end_line":55,"end_character":56},"updated":"2016-07-13 02:45:14.000000000","message":"\"allows us to specify\" or \"allows specifying\"","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":62,"context_line":"had provided."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":65,"context_line":" - live-migration"},{"line_number":66,"context_line":" - cold-migration"},{"line_number":67,"context_line":" - evacuation aka resurrection (rebuild)"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_193c0192","line":65,"range":{"start_line":65,"start_character":0,"end_line":65,"end_character":17},"updated":"2016-07-13 02:45:14.000000000","message":"Is live migration relevant here?\n\nYou can\u0027t do a live migration unless the source is alive and reachable which I didn\u0027t think was ever the case in these scenarios.","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"83944f5af3544bc3436d005cd4a2128745863bf2","unresolved":false,"context_lines":[{"line_number":86,"context_line":"* Use Pacemaker remote [#]_, fence agents [#]_, Masakari [#]_ auto-evacuation"},{"line_number":87,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":88,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":89,"context_line":"* Use DLM from Tooz [#]_ complementary to STONITH based fencing. Technically,"},{"line_number":90,"context_line":"  DLM may guarantee a critical section to be held exclusively without power"},{"line_number":91,"context_line":"  management or resources isolation related fencing actions to the cluster."},{"line_number":92,"context_line":"  For example, there is no STONITH in a Zookeeper, instead there are fencing"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_b4b17449","line":89,"updated":"2016-07-13 02:45:14.000000000","message":"Its unclear to me what a DLM is intended to be an alternative to in this context.\nUnless it was intended to be an optional addition to the preceding alternative?","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"9b5ea9d94496429150cbf68fc54d254df072130b","unresolved":false,"context_lines":[{"line_number":91,"context_line":"  management or resources isolation related fencing actions to the cluster."},{"line_number":92,"context_line":"  For example, there is no STONITH in a Zookeeper, instead there are fencing"},{"line_number":93,"context_line":"  tokens [#]_ that prevent data from unexpected changes. But DLM cannot cover"},{"line_number":94,"context_line":"  all cases, for example VMs running via libvirt, with access to a shared"},{"line_number":95,"context_line":"  storage, which has no clue about fencing tokens."},{"line_number":96,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":97,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_00ab8b1e","line":94,"updated":"2016-04-16 12:11:47.000000000","message":"Which other cases would a DLM cover which are relevant to this spec?","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"604e322fc951963e96eb1e954a0c03f60624bdc4","unresolved":false,"context_lines":[{"line_number":91,"context_line":"  management or resources isolation related fencing actions to the cluster."},{"line_number":92,"context_line":"  For example, there is no STONITH in a Zookeeper, instead there are fencing"},{"line_number":93,"context_line":"  tokens [#]_ that prevent data from unexpected changes. But DLM cannot cover"},{"line_number":94,"context_line":"  all cases, for example VMs running via libvirt, with access to a shared"},{"line_number":95,"context_line":"  storage, which has no clue about fencing tokens."},{"line_number":96,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":97,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"1aa78d24_d8d38a02","line":94,"in_reply_to":"1a122d0e_00ab8b1e","updated":"2016-07-13 08:14:23.000000000","message":"Perhaps DLM is the only choice when computes are containers and ifra nodes cannot be accessed by STONITH agents (think of a GKE/GCE is hosting your OpenStack cloud as Kubernetes workloads)","commit_id":"f9ec3a0ef24dfa3948241e9033a44c8ac53d9281"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":1,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":2,"context_line":" Instances auto evacuation"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"OpenStack project requires unified auto-evacuation mechanism."}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_435cbe7c","line":2,"range":{"start_line":2,"start_character":16,"end_line":2,"end_character":26},"updated":"2016-07-21 16:29:23.000000000","message":"In general I think it is a bad idea to use the word \"evacuation\" in this context, for reasons already explained here: https://youtu.be/lddtWUP_IKQ?t\u003d13m09s\n\n\"Resurrection\" is a more appropriate choice of word, because a) it is more accurate, and b) it focuses on the ultimate goal, which is recovery of workloads.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Problem description"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Host evacuation (instances resurrection) is an essential cloud platforms"},{"line_number":12,"context_line":"feature, which allows to reduce instance downtime. On relatively big"},{"line_number":13,"context_line":"environments it could be hard to track all potential problems and make workloads"},{"line_number":14,"context_line":"evacuate in time. OpenStack project already has instruments for gathering"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_43a15e45","line":11,"range":{"start_line":11,"start_character":0,"end_line":11,"end_character":40},"updated":"2016-07-21 16:29:23.000000000","message":"As mentioned above, these terms mean different things, so it increases the confusion by using them to refer to the same thing.  Host evacuation is more sensibly interpreted as live migration of instances away from a still healthy host.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":12,"context_line":"feature, which allows to reduce instance downtime. On relatively big"},{"line_number":13,"context_line":"environments it could be hard to track all potential problems and make workloads"},{"line_number":14,"context_line":"evacuate in time. OpenStack project already has instruments for gathering"},{"line_number":15,"context_line":"statistics, migration workloads, and workflow orchestration [#]_. While it"},{"line_number":16,"context_line":"possible to make workload migrate manually, OpenStack needs a unified approach"},{"line_number":17,"context_line":"for workload auto-evacuation. Proposed functionality should enable automatic"},{"line_number":18,"context_line":"detection of potential problems and evacuate instances from suffering node."}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_e3da1216","line":15,"range":{"start_line":15,"start_character":72,"end_line":15,"end_character":74},"updated":"2016-07-21 16:29:23.000000000","message":"\"it is\"","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":15,"context_line":"statistics, migration workloads, and workflow orchestration [#]_. While it"},{"line_number":16,"context_line":"possible to make workload migrate manually, OpenStack needs a unified approach"},{"line_number":17,"context_line":"for workload auto-evacuation. Proposed functionality should enable automatic"},{"line_number":18,"context_line":"detection of potential problems and evacuate instances from suffering node."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_636c6208","line":18,"range":{"start_line":18,"start_character":60,"end_line":18,"end_character":69},"updated":"2016-07-21 16:29:23.000000000","message":"\"suffering\" is a bit ambiguous.  A node could be suffering from poor performance (e.g. due to bad tuning or overloading), but this is not the same as node failure.  It should be noted that the related user story:\n\nhttp://specs.openstack.org/openstack/openstack-user-stories/user-stories/proposed/ha_vm.html\n\nonly focuses on node failure; if we extend the scope to deal with things like poor performance then that makes the problem much more complicated to address, because then we need Ceilometer or similar for monitoring performance.\n\nSo personally I would recommend focusing on automating a reactive approach to hard *failures* for now.  Then later, when we have that implemented, we can widen our attention to building a *proactive* approach to \"soft failures\" (i.e. performance degradation and similar).  Otherwise this spec has a wider scope than the corresponding user story.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  +-------------------------------------------+"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Ceilometer has alarms abstraction that allows us to specify"},{"line_number":56,"context_line":"metrics to watch, threshold value, and uri that should be"},{"line_number":57,"context_line":"requested. When the metric meets threshold ceilometer notifies"},{"line_number":58,"context_line":"evacuation workflow controller that triggers the whole process."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_e357b223","line":56,"range":{"start_line":56,"start_character":39,"end_line":56,"end_character":42},"updated":"2016-07-21 16:29:23.000000000","message":"\"URI\"","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":55,"context_line":"Ceilometer has alarms abstraction that allows us to specify"},{"line_number":56,"context_line":"metrics to watch, threshold value, and uri that should be"},{"line_number":57,"context_line":"requested. When the metric meets threshold ceilometer notifies"},{"line_number":58,"context_line":"evacuation workflow controller that triggers the whole process."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Auto-evacuation controller makes decision, which type of instance migration"},{"line_number":61,"context_line":"is more preferable. Decision should be made based on info, that ceilometer"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_a64e6838","line":58,"updated":"2016-07-21 16:29:23.000000000","message":"As per above comment, I think we should keep things simple and initially avoid the need for monitoring / metrics / thresholds.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":62,"context_line":"had provided."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":65,"context_line":" - live-migration (optimisticly assumes the source is alive)"},{"line_number":66,"context_line":" - cold-migration"},{"line_number":67,"context_line":" - evacuation aka resurrection (rebuild)"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_832356bc","line":65,"range":{"start_line":65,"start_character":19,"end_line":65,"end_character":31},"updated":"2016-07-21 16:29:23.000000000","message":"\"optimistically\"","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Nova execute workload migration. Several options available:"},{"line_number":65,"context_line":" - live-migration (optimisticly assumes the source is alive)"},{"line_number":66,"context_line":" - cold-migration"},{"line_number":67,"context_line":" - evacuation aka resurrection (rebuild)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":".. note ::"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_06c0144f","line":66,"updated":"2016-07-21 16:29:23.000000000","message":"Yes, migration is required if we want to implement a proactive approach based on metrics.  That\u0027s another example of the additional complexity resulting from not limiting the scope to a reactive approach.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":76,"context_line":"* Not to introduce this functionality."},{"line_number":77,"context_line":"* Let users to make a preference on what happens to the VM on failure, which"},{"line_number":78,"context_line":"  could be exposed via the Nova API, like delete on failure, or reboot on"},{"line_number":79,"context_line":"  failure, or resurrect (auto-evac), or a suchlike."},{"line_number":80,"context_line":"* Use Heat instead of Mistral."},{"line_number":81,"context_line":"  TODO(bogdando) Elaborate, is Heat is capable of executing arbitrary"},{"line_number":82,"context_line":"  workflows, like evacuation?"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_46adec99","line":79,"range":{"start_line":79,"start_character":37,"end_line":79,"end_character":50},"updated":"2016-07-21 16:29:23.000000000","message":"\"or similar\"","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":79,"context_line":"  failure, or resurrect (auto-evac), or a suchlike."},{"line_number":80,"context_line":"* Use Heat instead of Mistral."},{"line_number":81,"context_line":"  TODO(bogdando) Elaborate, is Heat is capable of executing arbitrary"},{"line_number":82,"context_line":"  workflows, like evacuation?"},{"line_number":83,"context_line":"* Create API in nova for host auto-evacuation - nova scheduler could pick"},{"line_number":84,"context_line":"  appropriate host for instance being migrated, so nova could acts as workflow"},{"line_number":85,"context_line":"  controller. Ceilometer will execute nova host-evacuate then."}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_268b9820","line":82,"updated":"2016-07-21 16:29:23.000000000","message":"I don\u0027t think it is.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":86,"context_line":"* Use Pacemaker remote [#]_, fence agents [#]_, Masakari [#]_ auto-evacuation"},{"line_number":87,"context_line":"  controller and monitors and/or Mistral workflows instead of Ceilometer alarms"},{"line_number":88,"context_line":"  (these have been researched in the hope to became a main implementation path)."},{"line_number":89,"context_line":"* Use DLM from Tooz [#]_ complementary to STONITH based fencing, for example"},{"line_number":90,"context_line":"  when computes are containers and host nodes cannot be accessed by STONITH"},{"line_number":91,"context_line":"  agents (think of a GKE/GCE is hosting your OpenStack cloud as Kubernetes"},{"line_number":92,"context_line":"  workloads). Technically, DLM may guarantee ongoing operations to be held"},{"line_number":93,"context_line":"  exclusively without power management or resources isolation related fencing"},{"line_number":94,"context_line":"  actions to the cluster. For example, there is no STONITH in a Zookeeper,"},{"line_number":95,"context_line":"  instead there are fencing tokens [#]_ that prevent data from unexpected"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_0695b431","line":92,"range":{"start_line":89,"start_character":65,"end_line":92,"end_character":12},"updated":"2016-07-21 16:29:23.000000000","message":"Why would STONITH not be possible in this case?","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":93,"context_line":"  exclusively without power management or resources isolation related fencing"},{"line_number":94,"context_line":"  actions to the cluster. For example, there is no STONITH in a Zookeeper,"},{"line_number":95,"context_line":"  instead there are fencing tokens [#]_ that prevent data from unexpected"},{"line_number":96,"context_line":"  changes. But DLM cannot cover all cases, for example when VMs running via"},{"line_number":97,"context_line":"  libvirt with access to a shared storage, which has no clue about fencing"},{"line_number":98,"context_line":"  tokens."},{"line_number":99,"context_line":"* More details to the alternatives can be found in this etherpad [#]_."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_c6befcac","line":98,"range":{"start_line":96,"start_character":11,"end_line":98,"end_character":9},"updated":"2016-07-21 16:29:23.000000000","message":"Containers don\u0027t have a clue about fencing tokens either.  I can\u0027t possibly see how fencing tokens would work in this case - please can you explain that?","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"386d2daeb86c94dfb2dc8aac5a42e860b75f2da5","unresolved":false,"context_lines":[{"line_number":128,"context_line":"* Final solution to be done based on the Mistral HA and stability for"},{"line_number":129,"context_line":"  running workflows. By results, use either Masakari or Mistral or"},{"line_number":130,"context_line":"  a combination of both. For fencing use either Pacemaker with Pacemaker_remote"},{"line_number":131,"context_line":"  or DLM from Tooz for locking."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Dependencies"},{"line_number":134,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dada55a8_a6fb0878","line":131,"range":{"start_line":131,"start_character":5,"end_line":131,"end_character":30},"updated":"2016-07-21 16:29:23.000000000","message":"I\u0027m 99% sure this cannot work, but I\u0027m happy to be proven wrong if someone can explain it to me.","commit_id":"fe171d3e8aa1a78f71f7d61eb2ec6538a9c2c1ab"}]}