)]}'
{"specs/other/high-availability.rst":[{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"b98d3b80e449c208f395c26fa2f94dc22f6d1df9","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Primary assignee:"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"* Sampath Priyankara (samP at IRC)"},{"line_number":90,"context_line":"* Timur Nurlygayanov (tnurlygayanov@mirantis.com)"},{"line_number":91,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Mandatory design reviewers:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a30ddce_15ea810f","line":90,"updated":"2017-03-09 10:29:23.000000000","message":"I think we can remove Timur because he is working on another project. Correct me if I am not right","commit_id":"7159ba44a000f5784d9e28c370dae2cadaf0f417"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"f3f368f58d7cf34e36c8625a487988cb069a6b04","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Primary assignee:"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"* Sampath Priyankara (samP at IRC)"},{"line_number":90,"context_line":"* Timur Nurlygayanov (tnurlygayanov@mirantis.com)"},{"line_number":91,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Mandatory design reviewers:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_d9493eb7","line":90,"in_reply_to":"9a30ddce_15ea810f","updated":"2017-04-11 11:12:55.000000000","message":"Done","commit_id":"7159ba44a000f5784d9e28c370dae2cadaf0f417"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"aa63a49a1dc863207f63f29083f404be59081b48","unresolved":false,"context_lines":[{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"Stepler Test Framework"},{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"During the OpenStack summit in Barcelona we discussed not covered area for"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5ff73747_2271440d","line":11,"range":{"start_line":11,"start_character":0,"end_line":11,"end_character":7},"updated":"2017-04-27 02:40:58.000000000","message":"I am not sure why the name is Stepler..\nYou can name it as you like if having some idea with the reason :)","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"b55d3e8fe021441698ecf3371fbc3f02f396bcc7","unresolved":false,"context_lines":[{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"Stepler Test Framework"},{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"During the OpenStack summit in Barcelona we discussed not covered area for"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f201791_e913860c","line":11,"range":{"start_line":11,"start_character":0,"end_line":11,"end_character":7},"in_reply_to":"5ff73747_2271440d","updated":"2017-06-22 05:48:18.000000000","message":"Yeh, I got similar comments form few others.\nsuch as, word \"Stepler\" s \nLet\u0027s think a good name for it. Till then it will be,\n\"The Destructive Test Framework\"","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"aa63a49a1dc863207f63f29083f404be59081b48","unresolved":false,"context_lines":[{"line_number":17,"context_line":"We need to create a new framework which is not intended for OpenStack API"},{"line_number":18,"context_line":"validation, but provides simple tool for creation tests to check destructive"},{"line_number":19,"context_line":"scenarios which will check High Availability, failover and recovery testing"},{"line_number":20,"context_line":"of OpenStack pre-production clouds. This a part of the testing scenarios"},{"line_number":21,"context_line":"addressed in OpenStack Extreme Testing [1]_ user story."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5ff73747_02640849","line":20,"range":{"start_line":20,"start_character":36,"end_line":20,"end_character":42},"updated":"2017-04-27 02:40:58.000000000","message":"s/This a/This is a/","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"b55d3e8fe021441698ecf3371fbc3f02f396bcc7","unresolved":false,"context_lines":[{"line_number":17,"context_line":"We need to create a new framework which is not intended for OpenStack API"},{"line_number":18,"context_line":"validation, but provides simple tool for creation tests to check destructive"},{"line_number":19,"context_line":"scenarios which will check High Availability, failover and recovery testing"},{"line_number":20,"context_line":"of OpenStack pre-production clouds. This a part of the testing scenarios"},{"line_number":21,"context_line":"addressed in OpenStack Extreme Testing [1]_ user story."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f231b9d_a90f7335","line":20,"range":{"start_line":20,"start_character":36,"end_line":20,"end_character":42},"in_reply_to":"5ff73747_02640849","updated":"2017-06-22 05:48:18.000000000","message":"Done","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"aa63a49a1dc863207f63f29083f404be59081b48","unresolved":false,"context_lines":[{"line_number":14,"context_line":"During the OpenStack summit in Barcelona we discussed not covered area for"},{"line_number":15,"context_line":"OpenStack clouds testing [0]_."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"We need to create a new framework which is not intended for OpenStack API"},{"line_number":18,"context_line":"validation, but provides simple tool for creation tests to check destructive"},{"line_number":19,"context_line":"scenarios which will check High Availability, failover and recovery testing"},{"line_number":20,"context_line":"of OpenStack pre-production clouds. This a part of the testing scenarios"},{"line_number":21,"context_line":"addressed in OpenStack Extreme Testing [1]_ user story."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Problem description"},{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5ff73747_c259808e","line":21,"range":{"start_line":17,"start_character":0,"end_line":21,"end_character":55},"updated":"2017-04-27 02:40:58.000000000","message":"+1","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":11520,"name":"Jamey McCabe","email":"jamemcc@gmail.com","username":"jamemcc"},"change_message_id":"a3d6f605260941f2e5694d2314f0f0a5dc9a33f5","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Alternatives"},{"line_number":50,"context_line":"------------"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"* `HA Failure Test \u003chttps://github.com/avdhoot07/HA-Failure-TEST\u003e`_"},{"line_number":53,"context_line":"* `Cloud99 \u003chttps://github.com/cisco-oss-eng/Cloud99\u003e`_"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bff0334d_d1cd8eb7","line":51,"updated":"2017-04-11 13:54:49.000000000","message":"Lets put in more detail the differences between things that are already supplied by exiting tools and projects.","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"b55d3e8fe021441698ecf3371fbc3f02f396bcc7","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Alternatives"},{"line_number":50,"context_line":"------------"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"* `HA Failure Test \u003chttps://github.com/avdhoot07/HA-Failure-TEST\u003e`_"},{"line_number":53,"context_line":"* `Cloud99 \u003chttps://github.com/cisco-oss-eng/Cloud99\u003e`_"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f201791_44a141a9","line":51,"in_reply_to":"bff0334d_d1cd8eb7","updated":"2017-06-22 05:48:18.000000000","message":"Done","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"aa63a49a1dc863207f63f29083f404be59081b48","unresolved":false,"context_lines":[{"line_number":60,"context_line":"Each of steps ends with it\u0027s own check, that step was finished correct."},{"line_number":61,"context_line":"It allows to compose plenty of tests, having moderate codebase."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"This framework should use os-faults library [2]_ to manage the nodes and"},{"line_number":64,"context_line":"networking switches (shutdown, run bash commands,"},{"line_number":65,"context_line":"restart services and etc.). os-fault library operates with 2 types of"},{"line_number":66,"context_line":"objects: service and nodes. It can inject faults in the form of"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5ff73747_c2ce202c","line":63,"range":{"start_line":63,"start_character":26,"end_line":63,"end_character":43},"updated":"2017-04-27 02:40:58.000000000","message":"I think we don\u0027t need to depend on the os-faults library now, because I guess its library could be out of maintenance.\nIt would be nice to avoid describing actual name on this spec and write the basic design instead as a first step.","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"b55d3e8fe021441698ecf3371fbc3f02f396bcc7","unresolved":false,"context_lines":[{"line_number":60,"context_line":"Each of steps ends with it\u0027s own check, that step was finished correct."},{"line_number":61,"context_line":"It allows to compose plenty of tests, having moderate codebase."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"This framework should use os-faults library [2]_ to manage the nodes and"},{"line_number":64,"context_line":"networking switches (shutdown, run bash commands,"},{"line_number":65,"context_line":"restart services and etc.). os-fault library operates with 2 types of"},{"line_number":66,"context_line":"objects: service and nodes. It can inject faults in the form of"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f201791_e5097ddd","line":63,"range":{"start_line":63,"start_character":26,"end_line":63,"end_character":43},"in_reply_to":"5ff73747_c2ce202c","updated":"2017-06-22 05:48:18.000000000","message":"Thank you. I will address them in more general way.\nSeems like you are correct about os-faults project. \nHowever, IMO os-faults has lot more good features we could use.\nSo, I do not hesitate to reactivate the project.","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"aa63a49a1dc863207f63f29083f404be59081b48","unresolved":false,"context_lines":[{"line_number":66,"context_line":"objects: service and nodes. It can inject faults in the form of"},{"line_number":67,"context_line":"Service-oriented, Node-oriented or Network-oriented commands to"},{"line_number":68,"context_line":"perform specified action on specified node(s)."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Architecture should have the following abstraction levels of codebase (from"},{"line_number":71,"context_line":"higher to less):"},{"line_number":72,"context_line":" - clients are able to manipulate resources: users, roles, servers, etc."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5ff73747_0265281e","line":69,"updated":"2017-04-27 02:40:58.000000000","message":"More important thing is that this kind of HA depends on how to deploy OpenStack. That means which distribution is used, which software is used and how many nodes are used on a testing target cloud.\nAs you know, there is a lot of OpenStack distributions in the world. So if testing this framework, which options could be expected at this time?\nOr do we need to say we will not test this framework with actual deployments(like Devstack) at the gate?\n\nAnyways, it is necessary to clarify how to test this framework as an integration test on the gate in this spec.","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"b55d3e8fe021441698ecf3371fbc3f02f396bcc7","unresolved":false,"context_lines":[{"line_number":66,"context_line":"objects: service and nodes. It can inject faults in the form of"},{"line_number":67,"context_line":"Service-oriented, Node-oriented or Network-oriented commands to"},{"line_number":68,"context_line":"perform specified action on specified node(s)."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Architecture should have the following abstraction levels of codebase (from"},{"line_number":71,"context_line":"higher to less):"},{"line_number":72,"context_line":" - clients are able to manipulate resources: users, roles, servers, etc."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f201791_09b6c2fb","line":69,"in_reply_to":"5ff73747_0265281e","updated":"2017-06-22 05:48:18.000000000","message":"I add more details regarding this. please check the next patch.","commit_id":"f0c19d811db7ff8628de64dfd6e9284442b7744b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":5,"context_line":""},{"line_number":6,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":7,"context_line":""},{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"Stepler Test Framework"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df3967d1_17da8116","line":8,"range":{"start_line":8,"start_character":2,"end_line":8,"end_character":2},"updated":"2017-07-31 05:19:56.000000000","message":"we need a LP BP also here","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":7,"context_line":""},{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"Stepler Test Framework"},{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":""},{"line_number":15,"context_line":".. TODO:: Find a good name"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ff346bd7_01d998f1","line":12,"range":{"start_line":10,"start_character":22,"end_line":12,"end_character":31},"updated":"2017-07-31 05:19:56.000000000","message":"need alignment here","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":7,"context_line":""},{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"Stepler Test Framework"},{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":""},{"line_number":15,"context_line":".. TODO:: Find a good name"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_9b945b46","line":12,"range":{"start_line":10,"start_character":22,"end_line":12,"end_character":31},"in_reply_to":"ff346bd7_01d998f1","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":""},{"line_number":15,"context_line":".. TODO:: Find a good name"},{"line_number":16,"context_line":"During the OpenStack summit in Barcelona we discussed not covered area for"},{"line_number":17,"context_line":"OpenStack clouds testing [0]_."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"df3967d1_5727e913","line":15,"range":{"start_line":15,"start_character":0,"end_line":15,"end_character":26},"updated":"2017-07-31 05:19:56.000000000","message":"other than stepler?","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":""},{"line_number":15,"context_line":".. TODO:: Find a good name"},{"line_number":16,"context_line":"During the OpenStack summit in Barcelona we discussed not covered area for"},{"line_number":17,"context_line":"OpenStack clouds testing [0]_."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_5bacb38b","line":15,"range":{"start_line":15,"start_character":0,"end_line":15,"end_character":26},"in_reply_to":"df3967d1_5727e913","updated":"2017-08-29 14:44:51.000000000","message":"Not sure we can reuse the name \"stepler\" as it is used by Mirantis in [1]. However, stepler codes [1] are under Apache License, Version 2.0. I would like to rename this to \"OpenStack Destructive Test (OSDT) Framework\".\n[1] https://github.com/Mirantis/stepler","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"598e5835c0cea1b5a44f7dda49401cdb4216ad9f","unresolved":false,"context_lines":[{"line_number":35,"context_line":"What we want to check and cover with these tests:"},{"line_number":36,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":37,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":38,"context_line":" - Check the proper work of auto-healing scripts / pacemaker / OCF and etc."},{"line_number":39,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":40,"context_line":"   nodes / services."},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_22bc3068","line":38,"range":{"start_line":38,"start_character":28,"end_line":38,"end_character":40},"updated":"2017-07-12 10:26:29.000000000","message":"Sorry What is auto-healing? is it auto-scaling?","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"0838b8f18c4b897aef6e994f39d6523cab6c0fd8","unresolved":false,"context_lines":[{"line_number":35,"context_line":"What we want to check and cover with these tests:"},{"line_number":36,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":37,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":38,"context_line":" - Check the proper work of auto-healing scripts / pacemaker / OCF and etc."},{"line_number":39,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":40,"context_line":"   nodes / services."},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_28264fb9","line":38,"range":{"start_line":38,"start_character":28,"end_line":38,"end_character":40},"in_reply_to":"1f1a1f67_22bc3068","updated":"2017-07-12 10:59:32.000000000","message":"Thanks for the review and comment.\nSome operators uses custom scripts to repair or clean up the failed processes or nodes. Those scripts would be log clean up or backup, service restarts and cleaning rabbitmq queues, clean other garbage,.etc. And it is also possible to use custom scripts to define the fail-over methods and recovery method\nfor service/node or any other resources in the cloud.\nIn this spec, I address them all as \"auto-healing scripts\".","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"233be1c974bbccaa46a302dc522b2062a6860856","unresolved":false,"context_lines":[{"line_number":35,"context_line":"What we want to check and cover with these tests:"},{"line_number":36,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":37,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":38,"context_line":" - Check the proper work of auto-healing scripts / pacemaker / OCF and etc."},{"line_number":39,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":40,"context_line":"   nodes / services."},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_b41295b0","line":38,"range":{"start_line":38,"start_character":28,"end_line":38,"end_character":40},"in_reply_to":"1f1a1f67_28264fb9","updated":"2017-07-13 08:40:43.000000000","message":"I will include this details in next update..","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":35,"context_line":"What we want to check and cover with these tests:"},{"line_number":36,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":37,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":38,"context_line":" - Check the proper work of auto-healing scripts / pacemaker / OCF and etc."},{"line_number":39,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":40,"context_line":"   nodes / services."},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_1930e5e7","line":38,"range":{"start_line":38,"start_character":28,"end_line":38,"end_character":40},"in_reply_to":"1f1a1f67_b41295b0","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":44,"context_line":"Proposed change"},{"line_number":45,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Add new repository with framework to provide High Availability, destructive,"},{"line_number":48,"context_line":"failover, and recovery testing for control plane."},{"line_number":49,"context_line":"Add test cases based on its framework."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df3967d1_57948944","line":49,"range":{"start_line":47,"start_character":0,"end_line":49,"end_character":38},"updated":"2017-07-31 05:19:56.000000000","message":"need little bit more details here at least\n- will it be as tempest plugin\n- based on os-faults ? \n- it will have common framework as well as tests also.\netc\n\nNot arch but at least high level idea of this framework structure.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":44,"context_line":"Proposed change"},{"line_number":45,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Add new repository with framework to provide High Availability, destructive,"},{"line_number":48,"context_line":"failover, and recovery testing for control plane."},{"line_number":49,"context_line":"Add test cases based on its framework."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_174d49c3","line":49,"range":{"start_line":47,"start_character":0,"end_line":49,"end_character":38},"in_reply_to":"df3967d1_57948944","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"8e56b86c7487a9e06dd1b2418b6f7a317fdb1a3d","unresolved":false,"context_lines":[{"line_number":49,"context_line":"Add test cases based on its framework."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Alternatives"},{"line_number":53,"context_line":"------------"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"1. `Cloud99 \u003chttps://github.com/cisco-oss-eng/Cloud99\u003e`_"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df3967d1_ab3beee8","line":52,"updated":"2017-08-02 17:11:32.000000000","message":"Thanks for including the alternatives.  Please can you also explain why the proposed change was chosen instead of any of these alternatives?","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":49,"context_line":"Add test cases based on its framework."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Alternatives"},{"line_number":53,"context_line":"------------"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"1. `Cloud99 \u003chttps://github.com/cisco-oss-eng/Cloud99\u003e`_"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_eb42d4aa","line":52,"in_reply_to":"df3967d1_ab3beee8","updated":"2017-08-29 14:44:51.000000000","message":"For any alternatives, we need to extend it\u0027s fault injection, test evaluation, metrics gathering.. etc. Therefore, rather than extending something new, I think it is better to extend something in we can commonly used in openstack, such as os-fault.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":14614,"name":"Kyrylo Romanenko","display_name":"K.Romanenko","email":"romanenko-kv@hotmail.com","username":"kromanenko"},"change_message_id":"c33feba43699f723e936eaca7f2849af253eddbf","unresolved":false,"context_lines":[{"line_number":79,"context_line":"   All scripts are located in FUEL server and will also be executed from"},{"line_number":80,"context_line":"   FUEL server only."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"3. `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_"},{"line_number":83,"context_line":"   This library provides an abstraction layer over different types of cloud"},{"line_number":84,"context_line":"   deployments. The actions are implemented as drivers (e.g. DevStack driver,"},{"line_number":85,"context_line":"   Fuel driver, Libvirt driver, IPMI driver). os-fault library operates with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_6a7e286c","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":13},"updated":"2017-07-14 09:51:11.000000000","message":"Actually os_faults is not an alternative but a requirement for Stepler framework. I suppose we should mention about OS Faults in `Implementation` section.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"724b6b151f2148c04136e54f87b0030cec189181","unresolved":false,"context_lines":[{"line_number":79,"context_line":"   All scripts are located in FUEL server and will also be executed from"},{"line_number":80,"context_line":"   FUEL server only."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"3. `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_"},{"line_number":83,"context_line":"   This library provides an abstraction layer over different types of cloud"},{"line_number":84,"context_line":"   deployments. The actions are implemented as drivers (e.g. DevStack driver,"},{"line_number":85,"context_line":"   Fuel driver, Libvirt driver, IPMI driver). os-fault library operates with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_c599af79","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":13},"in_reply_to":"1f1a1f67_6a7e286c","updated":"2017-07-14 10:48:55.000000000","message":"Thank you for the review.\nAgree, os_faults is a requirement for Stepler framework.\nHowever, please see Ken\u0027ichi Ohmichi\u0027s comment on\n(Apr 27 11:40 AM). I would like to discuss basic design\nas the first step. And I am drafting another spec for\ndiscuss tools and architecture of the framework.\nWe can discuss those in details, once we done here.\n# I should have cheanged the name \"Stepler\"","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":79,"context_line":"   All scripts are located in FUEL server and will also be executed from"},{"line_number":80,"context_line":"   FUEL server only."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"3. `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_"},{"line_number":83,"context_line":"   This library provides an abstraction layer over different types of cloud"},{"line_number":84,"context_line":"   deployments. The actions are implemented as drivers (e.g. DevStack driver,"},{"line_number":85,"context_line":"   Fuel driver, Libvirt driver, IPMI driver). os-fault library operates with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df3967d1_f7ec9df4","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":13},"in_reply_to":"1f1a1f67_c599af79","updated":"2017-07-31 05:19:56.000000000","message":"yea but better to not mention as alternate. If stepler going to be based on this then we should mention it in main part.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":79,"context_line":"   All scripts are located in FUEL server and will also be executed from"},{"line_number":80,"context_line":"   FUEL server only."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"3. `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_"},{"line_number":83,"context_line":"   This library provides an abstraction layer over different types of cloud"},{"line_number":84,"context_line":"   deployments. The actions are implemented as drivers (e.g. DevStack driver,"},{"line_number":85,"context_line":"   Fuel driver, Libvirt driver, IPMI driver). os-fault library operates with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_ab1a8cc8","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":13},"in_reply_to":"df3967d1_f7ec9df4","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":14614,"name":"Kyrylo Romanenko","display_name":"K.Romanenko","email":"romanenko-kv@hotmail.com","username":"kromanenko"},"change_message_id":"c33feba43699f723e936eaca7f2849af253eddbf","unresolved":false,"context_lines":[{"line_number":124,"context_line":"destruction."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"- Cat1: Destruction done with in the node, such as process down, I/O block,"},{"line_number":127,"context_line":"        service down, shutdown node via ssh, .etc."},{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_8fecf629","line":127,"range":{"start_line":127,"start_character":45,"end_line":127,"end_character":46},"updated":"2017-07-14 09:51:11.000000000","message":"This dot is not needed.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":124,"context_line":"destruction."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"- Cat1: Destruction done with in the node, such as process down, I/O block,"},{"line_number":127,"context_line":"        service down, shutdown node via ssh, .etc."},{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_4b29e8e3","line":127,"range":{"start_line":127,"start_character":45,"end_line":127,"end_character":46},"in_reply_to":"1f1a1f67_858df732","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"724b6b151f2148c04136e54f87b0030cec189181","unresolved":false,"context_lines":[{"line_number":124,"context_line":"destruction."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"- Cat1: Destruction done with in the node, such as process down, I/O block,"},{"line_number":127,"context_line":"        service down, shutdown node via ssh, .etc."},{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_858df732","line":127,"range":{"start_line":127,"start_character":45,"end_line":127,"end_character":46},"in_reply_to":"1f1a1f67_8fecf629","updated":"2017-07-14 10:48:55.000000000","message":"thanks, will update this in next patch.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":14614,"name":"Kyrylo Romanenko","display_name":"K.Romanenko","email":"romanenko-kv@hotmail.com","username":"kromanenko"},"change_message_id":"c33feba43699f723e936eaca7f2849af253eddbf","unresolved":false,"context_lines":[{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"},{"line_number":131,"context_line":"        network switches (shutdown, shutdown interface, restart..etc)"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Using this framework, destructive tests can be run after RC-1 release or"},{"line_number":134,"context_line":"periodically on OpenStack infrastructure or 3rd parties CI interface."}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_8f63d6b1","line":131,"range":{"start_line":131,"start_character":63,"end_line":131,"end_character":65},"updated":"2017-07-14 09:51:11.000000000","message":",","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"},{"line_number":131,"context_line":"        network switches (shutdown, shutdown interface, restart..etc)"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Using this framework, destructive tests can be run after RC-1 release or"},{"line_number":134,"context_line":"periodically on OpenStack infrastructure or 3rd parties CI interface."}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_0b1f60b7","line":131,"range":{"start_line":131,"start_character":63,"end_line":131,"end_character":65},"in_reply_to":"1f1a1f67_4587df10","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"724b6b151f2148c04136e54f87b0030cec189181","unresolved":false,"context_lines":[{"line_number":128,"context_line":"- Cat2: Destruction done to whole node, such as node power"},{"line_number":129,"context_line":"        on/off/reset/cycle/soft via IPMI"},{"line_number":130,"context_line":"- Cat3: Destruction done to external devices in infrastructure such as,"},{"line_number":131,"context_line":"        network switches (shutdown, shutdown interface, restart..etc)"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Using this framework, destructive tests can be run after RC-1 release or"},{"line_number":134,"context_line":"periodically on OpenStack infrastructure or 3rd parties CI interface."}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_4587df10","line":131,"range":{"start_line":131,"start_character":63,"end_line":131,"end_character":65},"in_reply_to":"1f1a1f67_8f63d6b1","updated":"2017-07-14 10:48:55.000000000","message":"thanks, will update this in next patch.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":14614,"name":"Kyrylo Romanenko","display_name":"K.Romanenko","email":"romanenko-kv@hotmail.com","username":"kromanenko"},"change_message_id":"c33feba43699f723e936eaca7f2849af253eddbf","unresolved":false,"context_lines":[{"line_number":148,"context_line":"Primary assignee:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_6f999a6c","line":151,"range":{"start_line":151,"start_character":0,"end_line":151,"end_character":40},"updated":"2017-07-14 09:51:11.000000000","message":"Sergei Chipiga already not in Mirantis and his current work is not connected with OpenStack.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":148,"context_line":"Primary assignee:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ff346bd7_24b02a43","line":151,"range":{"start_line":151,"start_character":0,"end_line":151,"end_character":40},"in_reply_to":"1f1a1f67_257e2319","updated":"2017-07-31 05:19:56.000000000","message":"yea, i can help on this","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"724b6b151f2148c04136e54f87b0030cec189181","unresolved":false,"context_lines":[{"line_number":148,"context_line":"Primary assignee:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1f1a1f67_257e2319","line":151,"range":{"start_line":151,"start_character":0,"end_line":151,"end_character":40},"in_reply_to":"1f1a1f67_6f999a6c","updated":"2017-07-14 10:48:55.000000000","message":"Thanks for info. I will remove him from assignee in next patch.","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":148,"context_line":"Primary assignee:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_4b8a08bf","line":151,"range":{"start_line":151,"start_character":0,"end_line":151,"end_character":40},"in_reply_to":"ff346bd7_24b02a43","updated":"2017-08-29 14:44:51.000000000","message":"Thanks. I add you here..","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"},{"line_number":155,"context_line":"* Luz Cazares (luzC at IRC)"},{"line_number":156,"context_line":"* Ken\u0027ichi Ohmichi (oomichi at IRC)"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"Milestones"},{"line_number":159,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ff346bd7_a4853a5c","line":156,"range":{"start_line":153,"start_character":0,"end_line":156,"end_character":35},"updated":"2017-07-31 05:19:56.000000000","message":"this really not needed, all QA members can help in review","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":150,"context_line":"* Sampath Priyankara (sampath.priyankara@lab.ntt.co.jp or samP at IRC)"},{"line_number":151,"context_line":"* Sergei Chipiga (schipiga@mirantis.com)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Mandatory design reviewers:"},{"line_number":154,"context_line":"* Dina Belova (Mirantis, DinaBelova at IRC)"},{"line_number":155,"context_line":"* Luz Cazares (luzC at IRC)"},{"line_number":156,"context_line":"* Ken\u0027ichi Ohmichi (oomichi at IRC)"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"Milestones"},{"line_number":159,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_2b87bcb7","line":156,"range":{"start_line":153,"start_character":0,"end_line":156,"end_character":35},"in_reply_to":"ff346bd7_a4853a5c","updated":"2017-08-29 14:44:51.000000000","message":"Done","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5bc7aeaee1b15bedf80fdc973861e237ca424cbe","unresolved":false,"context_lines":[{"line_number":159,"context_line":"----------"},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Target Milestone for completion:"},{"line_number":162,"context_line":"  Pike2"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Work Items"},{"line_number":165,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ff346bd7_84efd61c","line":162,"range":{"start_line":162,"start_character":0,"end_line":162,"end_character":7},"updated":"2017-07-31 05:19:56.000000000","message":"This needs to be updated","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":8716,"name":"Sampath Priyankara","email":"sam47priya@gmail.com","username":"sampath"},"change_message_id":"fbcc9b19d564ad34b801e9d15fbef88893c1a1d8","unresolved":false,"context_lines":[{"line_number":159,"context_line":"----------"},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Target Milestone for completion:"},{"line_number":162,"context_line":"  Pike2"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Work Items"},{"line_number":165,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_cb64b824","line":162,"range":{"start_line":162,"start_character":0,"end_line":162,"end_character":7},"in_reply_to":"ff346bd7_84efd61c","updated":"2017-08-29 14:44:51.000000000","message":"Just set it to Queens-1, But I do not think we could complete it before, Q-1. Nee to discuss this..","commit_id":"4c468991a970a5b108469d61fb8fb4b4d8e7b25b"},{"author":{"_account_id":26856,"name":"Gautam Divgi","email":"gautamdivgi@gmail.com"},"change_message_id":"8f974c4add475f64587e50c321a6ff5e4bc02c51","unresolved":false,"context_lines":[{"line_number":7,"context_line":""},{"line_number":8,"context_line":".."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":"OpenStack Destructive Test (OSDT) Framework"},{"line_number":12,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":13,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_487107a8","line":10,"updated":"2017-09-07 15:37:36.000000000","message":"Instead of OSDT I\u0027d like to propose the name Eris (after the Greek goddess of strife, discord and destruction)","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Problem description"},{"line_number":27,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"We need to test High Availability of OpenStack services and check how"},{"line_number":30,"context_line":"OpenStack services will work in case of any destructive scenarios."},{"line_number":31,"context_line":"Such tests allow to verify that OpenStack cluster will stay alive in"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_1e03ae11","line":28,"updated":"2017-09-03 16:42:17.000000000","message":"I think this may need a more specific definition.\n\nOpenStack services per-se are not high available, but they aim to have certain features that makes it possible to be deployed in a high available fashion. Testing that does not necessarily require an high-available deployment. For instance checking that API services are stateless can be tested by restarting the service in a one node deployment. \n\nThe other thing that you may want to test is that your deployment is high available / resilient to failure to the expected degree. This in fact is a test that your deployment / lifecycle tool of choice and your configuration management tools and your configuration are doing the right thing. \n\nSo these tests would probably be interesting ones for openstack-ansible, puppet, kolla \u0026 friends.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"What we want to check and cover with these tests:"},{"line_number":37,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":38,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":39,"context_line":" - Check the proper work of auto-healing scripts (custom scripts to repair or clean up the garbage of failed processes or nodes) / pacemaker / OCF and etc."},{"line_number":40,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":41,"context_line":"   nodes / services."}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_20088426","line":38,"updated":"2017-09-03 16:42:17.000000000","message":"stay alive?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":36,"context_line":"What we want to check and cover with these tests:"},{"line_number":37,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":38,"context_line":" - Will the cluster alive if we lose some nodes / services?"},{"line_number":39,"context_line":" - Check the proper work of auto-healing scripts (custom scripts to repair or clean up the garbage of failed processes or nodes) / pacemaker / OCF and etc."},{"line_number":40,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":41,"context_line":"   nodes / services."},{"line_number":42,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_e0115c55","line":39,"updated":"2017-09-03 16:42:17.000000000","message":"I\u0027m not sure where such auto-healing scripts live - are the part of an openstack deployment tool?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Other testing scenarios can be found in `Destructive testing \u003chttps://specs.openstack.org/openstack/openstack-user-stories/user-stories/proposed/openstack-extreme-testing.html#usage-scenario-examples\u003e`_ section in OpenStack Extreme Testing [1] user story."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Proposed change"},{"line_number":46,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Add new repository with framework to provide High Availability, destructive,"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_00746898","line":45,"updated":"2017-09-03 16:42:17.000000000","message":"I read this a few time but I think the spec is not really saying enough about what the proposed change is.\n\nIf proposed change is a framework that uses some existing tools, what does the framework provide exactly?\n\nOr is the spec about writing tests that use the tools listed below?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Add new repository with framework to provide High Availability, destructive,"},{"line_number":49,"context_line":"failover, and recovery testing for control plane and data plane."},{"line_number":50,"context_line":"Add test cases based on its framework."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"In this framework, we use following tools for load and failure injection."},{"line_number":53,"context_line":" - `Rally \u003chttps://wiki.openstack.org/wiki/Rally\u003e`_: For control plane load injection"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_a06234d7","line":50,"updated":"2017-09-03 16:42:17.000000000","message":"this framework?\n\nIt make sense to add tests to show how to use the framework, but it\u0027s not clear to me whether this spec is more about the framework or more about the tests?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":16425,"name":"Bin Bin Cong","email":"binbin.cong@easystack.cn","username":"stephen_cong"},"change_message_id":"eb69d6c1da7a13ebf0964b0c0eb0b2c94c3d01f9","unresolved":false,"context_lines":[{"line_number":49,"context_line":"failover, and recovery testing for control plane and data plane."},{"line_number":50,"context_line":"Add test cases based on its framework."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"In this framework, we use following tools for load and failure injection."},{"line_number":53,"context_line":" - `Rally \u003chttps://wiki.openstack.org/wiki/Rally\u003e`_: For control plane load injection"},{"line_number":54,"context_line":" - `Shaker \u003chttps://github.com/openstack/shaker\u003e`_: For data plane load injection"},{"line_number":55,"context_line":" - `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_: For failure injection"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_35c068e1","line":52,"updated":"2017-10-24 12:01:42.000000000","message":"How about using tempest to verify that all the api can work correctly when using os-faults to do fault injection?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"In this framework, we use following tools for load and failure injection."},{"line_number":53,"context_line":" - `Rally \u003chttps://wiki.openstack.org/wiki/Rally\u003e`_: For control plane load injection"},{"line_number":54,"context_line":" - `Shaker \u003chttps://github.com/openstack/shaker\u003e`_: For data plane load injection"},{"line_number":55,"context_line":" - `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_: For failure injection"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"os-faults library provides an abstraction layer over different types of cloud deployments."}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_20564408","line":54,"updated":"2017-09-03 16:42:17.000000000","message":"To use the terminology from the spec in [0] - today we do concurrent testing in our integration gate in the sense that we run multiple testing processes in parallel. However we do not necessarily test calls to specific APIs in parallel. I imagine this kind of testing might be something like issue a number of create/update/delete for a specific resource in \"parallel\" and verify that the end result is what expected. \n\nThe spec [0] talks about using Rally for this concurrent testing, but here you talk about load injection - which is a different thing. Are you thinking about load testing or concurrent testing or injecting APIs in the system while running HA scenarios?","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":55,"context_line":" - `os-faults \u003chttps://github.com/openstack/os-faults\u003e`_: For failure injection"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"os-faults library provides an abstraction layer over different types of cloud deployments."},{"line_number":58,"context_line":"The actions are implemented as drivers (e.g. DevStack driver, Fuel driver, Libvirt driver,"},{"line_number":59,"context_line":"IPMI driver). os-fault library operates with 2 types of objects: service"},{"line_number":60,"context_line":"and nodes. It can inject faults in the form of Service-oriented,"},{"line_number":61,"context_line":"Node-oriented or Network-oriented commands to perform specified action on"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_206f64c6","line":58,"updated":"2017-09-03 16:42:17.000000000","message":"Nice. A driver model is probably what we need since HA tests are very much deployment specific - a lot of what they test is the deployment itself in fact.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"7d53811c1a5e6c2655afcc7498c2cedecf5c583e","unresolved":false,"context_lines":[{"line_number":63,"context_line":"extend to preform other network switch failures and other external devices failures."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Alternatives"},{"line_number":67,"context_line":"------------"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"1. `Cloud99 \u003chttps://github.com/cisco-oss-eng/Cloud99\u003e`_"}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf57530f_9e36a2b4","line":66,"updated":"2017-09-20 09:00:18.000000000","message":"Could you please add Jepsen [0] framework to alternatives?\n\n[0] https://github.com/jepsen-io/jepsen","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":101,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"The test solution (framework) architecture should be based on"},{"line_number":104,"context_line":"STEPS-methodology, that considers a test as a sequence of steps."},{"line_number":105,"context_line":"Each of steps ends with it\u0027s own check, that step was finished correct."},{"line_number":106,"context_line":"It allows to compose plenty of tests, having moderate codebase."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_a0beb4fc","line":104,"updated":"2017-09-03 16:42:17.000000000","message":"One of the challenges of HA / resiliency testing is the fact that many different things need to happen in parallel: generate traffic on the system while injecting failures while probing the status.\n\nI wonder if the choice of a linear approach is the best for this case. I see the value in having re-usable \"bricks\" of tests to combine together but not necessarily in their strict sequential execution.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"ca4c827ec1a0bfa39e37e7ec67c906b9c1c794fc","unresolved":false,"context_lines":[{"line_number":101,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"The test solution (framework) architecture should be based on"},{"line_number":104,"context_line":"STEPS-methodology, that considers a test as a sequence of steps."},{"line_number":105,"context_line":"Each of steps ends with it\u0027s own check, that step was finished correct."},{"line_number":106,"context_line":"It allows to compose plenty of tests, having moderate codebase."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"1f9dbf25_aec27ae8","line":104,"in_reply_to":"7f287b81_a0beb4fc","updated":"2018-02-28 00:16:31.000000000","message":"It depends on the granularity of the bricks/steps. \n\nFor Pacemaker we have a series of tests (such as start/stop/kill a node/service) that were executed at random.  Sometimes these tests were combined, ie. a restart test wrapped the stop and start tests. \n\nEach test has pre and post conditions, but were definitely linear internally and validated progress at appropriate points.  Was each test a brick or a sequence of steps? \n\nUltimately you need elements of both.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"25925a602772323b10d4e12e4d9d72a711678cc0","unresolved":false,"context_lines":[{"line_number":119,"context_line":" - fixtures manage resources construction, destruction, etc via steps."},{"line_number":120,"context_line":" - tests combine steps and fixtures according to scenario."},{"line_number":121,"context_line":" - test cases are expected to live in the code tree of each project"},{"line_number":122,"context_line":"   such as, for nova, /nova/tests/destructive"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Destructive tests are expected to have longer run time than unit or"},{"line_number":125,"context_line":"functional tests. Therefore, execute full test scenarios for each patch"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7f287b81_a090f496","line":122,"updated":"2017-09-03 16:42:17.000000000","message":"I think it make sense for having some of this type of testing happening on single services - however I expect there will be the need for integration type of tests where we prove HA / resiliency to failure in a full cloud scenario.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":17517,"name":"Andrew Beekhof","email":"abeekhof@redhat.com","username":"beekhof"},"change_message_id":"ca4c827ec1a0bfa39e37e7ec67c906b9c1c794fc","unresolved":false,"context_lines":[{"line_number":119,"context_line":" - fixtures manage resources construction, destruction, etc via steps."},{"line_number":120,"context_line":" - tests combine steps and fixtures according to scenario."},{"line_number":121,"context_line":" - test cases are expected to live in the code tree of each project"},{"line_number":122,"context_line":"   such as, for nova, /nova/tests/destructive"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Destructive tests are expected to have longer run time than unit or"},{"line_number":125,"context_line":"functional tests. Therefore, execute full test scenarios for each patch"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1f9dbf25_0e760eda","line":122,"in_reply_to":"7f287b81_a090f496","updated":"2018-02-28 00:16:31.000000000","message":"If the service tests had predictable entry points (eg. ${service} start/stop/kill/status), then a higher level test could wrap them for use in more wholistic tests.","commit_id":"2c9f26f6b5818ad692433ff121861f4161f8ba8c"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":36,"context_line":"What we want to check and cover with these tests:"},{"line_number":37,"context_line":" - How OpenStack cluster behaves in case of some destructive scenarios?"},{"line_number":38,"context_line":" - Will the cluster stay alive if we lose some nodes / services?"},{"line_number":39,"context_line":" - Check the proper work of auto-healing scripts (custom scripts to repair or clean up the garbage of failed processes or nodes) / pacemaker / OCF and etc."},{"line_number":40,"context_line":" - Verify some OpenStack features which require shutdown / restart of"},{"line_number":41,"context_line":"   nodes / services."},{"line_number":42,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_7307701a","line":39,"updated":"2018-04-05 10:11:30.000000000","message":"Please wrap long lines, otherwise they are hard to review in Gerrit, and more likely to result in merge conflicts.","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Os-faults written in \"alternatives\" section is more useful and easy to learn than other "},{"line_number":64,"context_line":"failure injection tools because it is used in several OpenStack projects and has a strong affinity for them."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"However, it is intended for just causing some troubles, not for getting the status of targets and"},{"line_number":67,"context_line":"controlling them in detail."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"In a flexibility point of view, we decided to not use exisiting tools."},{"line_number":70,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_d308bcda","line":67,"range":{"start_line":66,"start_character":0,"end_line":67,"end_character":27},"updated":"2018-04-05 10:11:30.000000000","message":"But it\u0027s Open Source, so doesn\u0027t it make sense to extend it to encompass the extra functionality you need, instead of starting from scratch?  There are strong disadvantages to starting from scratch, e.g. it wastes time required to reimplement the same things, and it fragments a community which is already stretched thin on resources.\n\nIf there are deal-breaker reasons why os-faults can\u0027t be used here (e.g. an architectural mismatch, or you\u0027ve talked to the os-faults maintainers about extending the functionality and they refused to accept that direction) then please give details. Otherwise I\u0027m afraid it makes this decision sound like NIH syndrome, and I\u0027m sure that\u0027s not your intention.\n\nIs there a discussion on a mailing list somewhere about this, which can be linked to?","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":26856,"name":"Gautam Divgi","email":"gautamdivgi@gmail.com"},"change_message_id":"b654d327c020e9c6304623ac07798b36f088db0f","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Os-faults written in \"alternatives\" section is more useful and easy to learn than other "},{"line_number":64,"context_line":"failure injection tools because it is used in several OpenStack projects and has a strong affinity for them."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"However, it is intended for just causing some troubles, not for getting the status of targets and"},{"line_number":67,"context_line":"controlling them in detail."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"In a flexibility point of view, we decided to not use exisiting tools."},{"line_number":70,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_ea505e0d","line":67,"range":{"start_line":66,"start_character":0,"end_line":67,"end_character":27},"in_reply_to":"bf659307_d308bcda","updated":"2018-04-05 17:26:18.000000000","message":"I have some observations after trying out destructive testing in fairly large OpenStack environments. Not sure I want to put all these in a review here. But in summary:\n\n1. os-faults basically adds environment discovery, failure injection and a CLI and wraps them around Ansible programmatically. From my experience that\u0027s been too much code to write when extending the failures. What I\u0027ve been using is Ansible playbooks and dynamic inventories.\n\n2. Programmatic invocation of Ansible means we have to always keep up with Ansible internals. \n\n3. os-faults does provide some randomization around which nodes in a group to hit. E.g. you can say hit keystone and it will take a random keystone service out. But, this is easily done by creating Ansible strategy plugins as well.\n\nIn short, while os-faults was easy to start off with I essentially fell back to just using playbooks, dynamic inventories and custom created strategy plugins to randomize hits on targets. This approach has also made some of my modules reusable for the tenant space as well - e.g. I can use an Ansible module to hard-reboot a tenant VM by just using a different dynamic inventory (e.g. the OpenStack Ansible dynamic inventory script).","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"9c7e9cd6de1ce462038d1b8e59179c7d03156c2f","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Os-faults written in \"alternatives\" section is more useful and easy to learn than other "},{"line_number":64,"context_line":"failure injection tools because it is used in several OpenStack projects and has a strong affinity for them."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"However, it is intended for just causing some troubles, not for getting the status of targets and"},{"line_number":67,"context_line":"controlling them in detail."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"In a flexibility point of view, we decided to not use exisiting tools."},{"line_number":70,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e61f5007","line":67,"range":{"start_line":66,"start_character":0,"end_line":67,"end_character":27},"in_reply_to":"bf659307_ea505e0d","updated":"2018-04-26 13:55:31.000000000","message":"Thanks a lot - this is really helpful info, and I think it deserves to be included in the spec.  In the spirit of Open Design[0] I would definitely recommend opening a discussion with the community on this, just to make sure that nothing has been missed before deciding upon a single route.  For example, I\u0027ve just been made aware of http://chaostoolkit.org/ which looks like a great project, and I wonder if that could be harnessed.  The discussion can of course take place on the mailing lists and IRC, but I am also intending to organise a video call to kickstart collaboration between interested parties across the whole community:\n\nhttps://etherpad.openstack.org/p/extreme-testing-contacts\n\n[0] https://governance.openstack.org/tc/reference/opens.html#open-design","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":66,"context_line":"However, it is intended for just causing some troubles, not for getting the status of targets and"},{"line_number":67,"context_line":"controlling them in detail."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"In a flexibility point of view, we decided to not use exisiting tools."},{"line_number":70,"context_line":""},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_135074d4","line":69,"range":{"start_line":69,"start_character":54,"end_line":69,"end_character":63},"updated":"2018-04-05 10:11:30.000000000","message":"\"existing\"","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":26856,"name":"Gautam Divgi","email":"gautamdivgi@gmail.com"},"change_message_id":"b654d327c020e9c6304623ac07798b36f088db0f","unresolved":false,"context_lines":[{"line_number":109,"context_line":"   All scripts are located in FUEL server and will also be executed from"},{"line_number":110,"context_line":"   FUEL server only."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"4. `Jepsen \u003chttps://github.com/jepsen-io/jepsen\u003e`_"},{"line_number":113,"context_line":"   Jepsen is a tool intented for improving the safety of the distributed systems such"},{"line_number":114,"context_line":"   as databases, queues, consensus systems, and so on."},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_f498431c","line":112,"range":{"start_line":112,"start_character":49,"end_line":112,"end_character":50},"updated":"2018-04-05 17:26:18.000000000","message":"Jepsen is really a great tool. However, it is Clojure based. What I\u0027m worried about is expecting a community to pick up a new language and platform to be able to contribute here. This may be a very unfounded concern. But I thought I should bring it up.","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":110,"context_line":"   FUEL server only."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"4. `Jepsen \u003chttps://github.com/jepsen-io/jepsen\u003e`_"},{"line_number":113,"context_line":"   Jepsen is a tool intented for improving the safety of the distributed systems such"},{"line_number":114,"context_line":"   as databases, queues, consensus systems, and so on."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_f30060ae","line":113,"range":{"start_line":113,"start_character":20,"end_line":113,"end_character":28},"updated":"2018-04-05 10:11:30.000000000","message":"\"intended\", but since Jepsen is already a proven tool you can just drop this word: \"... is a tool for improving ...\"","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"942a627c2cd07f4c81088d079de6628489371902","unresolved":false,"context_lines":[{"line_number":111,"context_line":""},{"line_number":112,"context_line":"4. `Jepsen \u003chttps://github.com/jepsen-io/jepsen\u003e`_"},{"line_number":113,"context_line":"   Jepsen is a tool intented for improving the safety of the distributed systems such"},{"line_number":114,"context_line":"   as databases, queues, consensus systems, and so on."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Implementation"},{"line_number":117,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_b5861313","line":114,"updated":"2018-05-21 14:50:42.000000000","message":"thank you!","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":118,"context_line":""},{"line_number":119,"context_line":"The test solution (framework) architecture should be based on"},{"line_number":120,"context_line":"STEPS-methodology, that considers a test as a sequence of steps."},{"line_number":121,"context_line":"Each of steps ends with it\u0027s own check, that step was finished correct."},{"line_number":122,"context_line":"It allows to compose plenty of tests, having moderate codebase."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"This framework should be able to manage nodes, services and processes, and"}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_13fe34ae","line":121,"range":{"start_line":121,"start_character":24,"end_line":121,"end_character":28},"updated":"2018-04-05 10:11:30.000000000","message":"\"its\"","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"b2ffdc9789de72fb43a17b557c1361988a94dc0e","unresolved":false,"context_lines":[{"line_number":189,"context_line":"References"},{"line_number":190,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":191,"context_line":".. [0] `Etherpad from design session at Barcelona \u003chttps://etherpad.openstack.org/p/ocata-qa-destructive-testing\u003e`_"},{"line_number":192,"context_line":".. [1] `OpenStack Extream Testing user story \u003chttps://specs.openstack.org/openstack/openstack-user-stories/user-stories/proposed/openstack-extreme-testing.html\u003e`_"},{"line_number":193,"context_line":".. [2] `OS faults library \u003chttps://github.com/openstack/os-faults\u003e`_"}],"source_content_type":"text/x-rst","patch_set":8,"id":"bf659307_d3319ca4","line":192,"range":{"start_line":192,"start_character":18,"end_line":192,"end_character":25},"updated":"2018-04-05 10:11:30.000000000","message":"\"Extreme\"","commit_id":"578571dc303bfe4bb3a904193712d46424582bbb"}]}