)]}'
{"specs/other/openstack-upgrade-testing.rst":[{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know what projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, what projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally what projects support a seamless upgrade"},{"line_number":24,"context_line":"process that does not impact the performance of the cloud. To this end, the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_a24b9eae","line":21,"updated":"2017-03-23 20:09:24.000000000","message":"s/what/which","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know what projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, what projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally what projects support a seamless upgrade"},{"line_number":24,"context_line":"process that does not impact the performance of the cloud. To this end, the"},{"line_number":25,"context_line":"OpenStack TC defined tags that reflect the maturity of projects in terms of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_82c65a16","line":22,"updated":"2017-03-23 20:09:24.000000000","message":"s/what/which","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know what projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, what projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally what projects support a seamless upgrade"},{"line_number":24,"context_line":"process that does not impact the performance of the cloud. To this end, the"},{"line_number":25,"context_line":"OpenStack TC defined tags that reflect the maturity of projects in terms of"},{"line_number":26,"context_line":"their upgradeability, these tags are:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_a2e03e98","line":23,"updated":"2017-03-23 20:09:24.000000000","message":"s/what/which","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":34,"context_line":"designed with the main purpose of testing backward compatibility of project"},{"line_number":35,"context_line":"configuration, so it does not provide tests or tools for validating the"},{"line_number":36,"context_line":"different maturity levels of an upgrade. In order for projects to self assess"},{"line_number":37,"context_line":"their upgrade maturity level there is the need of such tests and tools."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As a first strategy to try to fill this gap there was a proposal for adding more"},{"line_number":40,"context_line":"tests and capabilities to Grenade, but the general feeling on the community was"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_82bd3a6a","line":37,"updated":"2017-03-23 20:09:24.000000000","message":"s/of such/for such","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":37,"context_line":"their upgrade maturity level there is the need of such tests and tools."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As a first strategy to try to fill this gap there was a proposal for adding more"},{"line_number":40,"context_line":"tests and capabilities to Grenade, but the general feeling on the community was"},{"line_number":41,"context_line":"that Grenade was already pushing its limits and it was preferable to fill this"},{"line_number":42,"context_line":"gap with a new tool or set of tools."},{"line_number":43,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_a2a71e50","line":40,"updated":"2017-03-23 20:09:24.000000000","message":"s/on the community/in the community","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"This spec proposes a new toolset for testing an OpenStack environment before,"},{"line_number":49,"context_line":"during, and after an upgrade process. The toolset answers the question \"how does"},{"line_number":50,"context_line":"OpenStack behaves across upgrades from one release N to a release N+1, or from"},{"line_number":51,"context_line":"the latest official release to master?\". It also provides information that can"},{"line_number":52,"context_line":"be used to assess if an upgrade complies with the requirements for being"},{"line_number":53,"context_line":"recognized as a `rolling upgrade`_, a `zero downtime upgrade`_ or a `zero impact"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_623596f7","line":50,"updated":"2017-03-23 20:09:24.000000000","message":"s/behaves/behave","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":50,"context_line":"OpenStack behaves across upgrades from one release N to a release N+1, or from"},{"line_number":51,"context_line":"the latest official release to master?\". It also provides information that can"},{"line_number":52,"context_line":"be used to assess if an upgrade complies with the requirements for being"},{"line_number":53,"context_line":"recognized as a `rolling upgrade`_, a `zero downtime upgrade`_ or a `zero impact"},{"line_number":54,"context_line":"upgrade`_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"The toolset will provide the necessary tools for testing the behavior of an"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_e22086b3","line":53,"updated":"2017-03-23 20:09:24.000000000","message":"formatting on zero impact upgrade is confused","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":68,"context_line":"actions."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"This toolset will be able to be used in CI systems as well as in a computer"},{"line_number":71,"context_line":"locally, so developers will be able to leverage on its tools."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_05401435","line":71,"updated":"2017-03-23 20:09:24.000000000","message":"...so developers will be able to leverage it in a development environment","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":200,"context_line":"Milestones"},{"line_number":201,"context_line":"----------"},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"The constructions of this toolset is a large effort, we will be building it"},{"line_number":204,"context_line":"incrementally during more than one cycle. The milestones for Pike are the to"},{"line_number":205,"context_line":"implement the following modules:"},{"line_number":206,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_c5b7cc3a","line":203,"updated":"2017-03-23 20:09:24.000000000","message":"s/constructions/construction","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"6d416f1a34a9e838b0f4df1470ccce6cf0521105","unresolved":false,"context_lines":[{"line_number":201,"context_line":"----------"},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"The constructions of this toolset is a large effort, we will be building it"},{"line_number":204,"context_line":"incrementally during more than one cycle. The milestones for Pike are the to"},{"line_number":205,"context_line":"implement the following modules:"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"- Test Manager"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ffe62b97_65880071","line":204,"updated":"2017-03-23 20:09:24.000000000","message":"s/the to/to","commit_id":"1eed07a4d2197df3682680f21a819e1d4da3ea6e"},{"author":{"_account_id":17716,"name":"Luz Cazares","email":"luz.cazares@intel.com","username":"luzcazares"},"change_message_id":"0b69aa47420f34f3d363d8e0e1c433690c7cfb0a","unresolved":false,"context_lines":[{"line_number":9,"context_line":" OpenStack Upgrade Test Toolset"},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"This spec describes the creation of a new toolset that could be used to test"},{"line_number":13,"context_line":"upgrades of OpenStack services."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_35ab188a","line":12,"range":{"start_line":12,"start_character":0,"end_line":12,"end_character":1},"updated":"2017-03-24 19:50:23.000000000","message":"Add \"As the OpenStack community works to improve the upgrade experience; there is a growing need to provide a variety of test tools which helps to exercise, harness and validate the upgrade approaches defined by the community.\" This spec...","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"f323fa15aafbf4fcd83fc7eac2ecbf0f42d4b912","unresolved":false,"context_lines":[{"line_number":9,"context_line":" OpenStack Upgrade Test Toolset"},{"line_number":10,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"This spec describes the creation of a new toolset that could be used to test"},{"line_number":13,"context_line":"upgrades of OpenStack services."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_157b26be","line":12,"range":{"start_line":12,"start_character":0,"end_line":12,"end_character":1},"in_reply_to":"ffe62b97_35ab188a","updated":"2017-03-27 20:56:44.000000000","message":"Sounds good, will add this in the next patch set","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":17716,"name":"Luz Cazares","email":"luz.cazares@intel.com","username":"luzcazares"},"change_message_id":"0b69aa47420f34f3d363d8e0e1c433690c7cfb0a","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Problem Description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know which projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, which projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally which projects support a seamless"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_35e798e7","line":20,"range":{"start_line":19,"start_character":1,"end_line":20,"end_character":58},"updated":"2017-03-24 19:50:23.000000000","message":"Replace with:\n\"OpenStack is a distributed system composed of several projects. Each project represent a service of the system and such service is deployed across a given infrastructure (physical or virtual) -.\n\nWe could probably omit that line.","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"f323fa15aafbf4fcd83fc7eac2ecbf0f42d4b912","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Problem Description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know which projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, which projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally which projects support a seamless"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_1bc4775e","line":20,"range":{"start_line":19,"start_character":1,"end_line":20,"end_character":58},"in_reply_to":"ffe62b97_35e798e7","updated":"2017-03-27 20:56:44.000000000","message":"With this text I\u0027m trying to give a background of the existence of upgrade tags. And actually I borrowed all this text in this paragraph from:\n\nhttps://governance.openstack.org/tc/reference/tags/assert_supports-rolling-upgrade.html#rationale\nhttps://governance.openstack.org/tc/reference/tags/assert_supports-zero-downtime-upgrade.html#rationale\nhttps://governance.openstack.org/tc/reference/tags/assert_supports-upgrade.html#rationale\n\nWhich is text that has already been reviewed and approved.","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":17716,"name":"Luz Cazares","email":"luz.cazares@intel.com","username":"luzcazares"},"change_message_id":"0b69aa47420f34f3d363d8e0e1c433690c7cfb0a","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know which projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, which projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally which projects support a seamless"},{"line_number":24,"context_line":"upgrade process that does not impact the performance of the cloud. To this end,"},{"line_number":25,"context_line":"the OpenStack TC defined tags that reflect the maturity of projects in terms of"},{"line_number":26,"context_line":"their upgradeability, these tags are:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"- `supports-rolling-upgrade`_"},{"line_number":29,"context_line":"- `supports-zero-downtime-upgrade`_"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_80e3c033","line":26,"range":{"start_line":20,"start_character":59,"end_line":26,"end_character":37},"updated":"2017-03-24 19:50:23.000000000","message":"While projects transitioned towards an upgrade methodology with minimal or no impact in their service; it become clear the need of additional levels of granularity, to match the difficulty of the implementation. As a result the TC defined the next project tags:","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"f323fa15aafbf4fcd83fc7eac2ecbf0f42d4b912","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"OpenStack projects represent code that is installed and deployed on many"},{"line_number":20,"context_line":"distributed systems in order to provide services to users. Operators need to"},{"line_number":21,"context_line":"know which projects support a rolling upgrade process to avoid significant"},{"line_number":22,"context_line":"downtime, which projects support an upgrade process that eliminates downtime of"},{"line_number":23,"context_line":"the control plane entirely, and finally which projects support a seamless"},{"line_number":24,"context_line":"upgrade process that does not impact the performance of the cloud. To this end,"},{"line_number":25,"context_line":"the OpenStack TC defined tags that reflect the maturity of projects in terms of"},{"line_number":26,"context_line":"their upgradeability, these tags are:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"- `supports-rolling-upgrade`_"},{"line_number":29,"context_line":"- `supports-zero-downtime-upgrade`_"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ffe62b97_9bad279d","line":26,"range":{"start_line":20,"start_character":59,"end_line":26,"end_character":37},"in_reply_to":"ffe62b97_80e3c033","updated":"2017-03-27 20:56:44.000000000","message":"Same comment as above.","commit_id":"da69ada3194e5f2507260ca27006708fd01c0eb6"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"b32dbcdda9504f774a6769925faea75cd3546d70","unresolved":false,"context_lines":[{"line_number":64,"context_line":"In a future stage, we could add a module to the toolset that would orchestrate"},{"line_number":65,"context_line":"the deployment and upgrade process of OpenStack. When this happens, this toolset"},{"line_number":66,"context_line":"could effectively replace Grenade since it would be able to perform the same"},{"line_number":67,"context_line":"validations Grenade does plus much more."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"To provide all the different tests an upgrade requires, the toolset could"},{"line_number":70,"context_line":"leverage from existing projects, like Tempest or Rally, to perform certain"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_2f33fc63","line":67,"updated":"2017-03-30 06:28:35.000000000","message":"I think we still need Grenade in the future stage. Because I think we should test OpenStack upgrade in a basic tooling like a bash script. Otherwise, it might be hard to debug. Of course, the toolset will be really reliable, we will be able to replace it, though.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"7d1aa1d6239df0980cbb7a5dc35daa1111273aab","unresolved":false,"context_lines":[{"line_number":64,"context_line":"In a future stage, we could add a module to the toolset that would orchestrate"},{"line_number":65,"context_line":"the deployment and upgrade process of OpenStack. When this happens, this toolset"},{"line_number":66,"context_line":"could effectively replace Grenade since it would be able to perform the same"},{"line_number":67,"context_line":"validations Grenade does plus much more."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"To provide all the different tests an upgrade requires, the toolset could"},{"line_number":70,"context_line":"leverage from existing projects, like Tempest or Rally, to perform certain"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_845ad480","line":67,"in_reply_to":"ffe62b97_2f33fc63","updated":"2017-03-30 16:14:26.000000000","message":"The purpose of this toolset is to test upgrades. At first it would cover those areas that are not being covered by Grenade, but in a future stage it could include the same validations Grenade is doing so we could have a single \"go to\" project to test upgrades. But of course, as you mentioned, when/if the time comes, we would need to evaluate its maturity and reliability.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":96,"context_line":"Requirements"},{"line_number":97,"context_line":"------------"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"**Functional**"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_25cb0273","line":99,"range":{"start_line":99,"start_character":0,"end_line":99,"end_character":14},"updated":"2017-03-30 16:49:49.000000000","message":"I understand that this aims to be a generic set of requirements with no constraint on the resulting architecture, but I find it a bit difficult to picture how this may look like. \n\nI think you need to define at least a bit more separation of responsibilities.\nWhat I understand now is that:\n- there is a tool that does initial deployment, not in scope for this spec\n- there is a tool that runs tests against a running system, not in scope for this spec\n- what is responsible for the actual update of each service? is the same tool used for deploy providing upgrade helpers as well?\n\nI guess the orchestration of the upgrade is responsibility of this tool? \nOne of the main thing that would be interesting to capture, that we don\u0027t see know with grenade is the cross-project dependencies / orchestration required to achieve \nzero down time. \n\nThis orchestration required to achieve zero downtime / rolling upgrade between two versions of OpenStack is kind of an extra step list, similar to what we have in grenade today, where we aim for the list of steps to be as small as possible.\nThis is one of the main advantages I see in testing upgrade, to ensure the life of operators and users for upgrades is not a nightmare, and to help us discover issues in this area as early as possible during the release cycle.\n\nOrchestration steps required are likely to depend on:\n- to/from versions in the upgrade process ( n -\u003e n+1 )\n- deployment architecture of the cloud to be upgraded\n\nThe latter is quite interesting because if we can prove that certain deployment architectures are better for seamless update, operators may decide to adopt that architecture as well.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":96,"context_line":"Requirements"},{"line_number":97,"context_line":"------------"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"**Functional**"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_cd41c83b","line":99,"range":{"start_line":99,"start_character":0,"end_line":99,"end_character":14},"in_reply_to":"ffe62b97_25cb0273","updated":"2017-04-03 22:12:58.000000000","message":"I will try to expand the Spec based on your comments here so it is clearer to everyone in the next patch, but here is some more explanation. \n\nTo better understand this spec is very important to consider the use cases we have outlined in it: \n\n- The first use case states that the tools from the toolset can be used by anyone (a user, a downstream project, a job in the gate, etc) to perform validations on a cloud, they can choose to use as many tools from the toolset and  in any order they want. Since they would be selecting what tools to run, and when, the toolset won\u0027t care if this is as part of an upgrade or not.  \n\n- The second use case is about having a partial control of the deployment/upgrade process, and a full control of the testing from the toolset, similar to what Grenade does. The toolset would be able to communicate with a deployment tool through a driver (e.g. OpenStack ansible), let\u0027s call this an orchestrator, so the toolset would be able to signal this orchestrator to deploy OpenStack, would run tests in a particular order, then would signal the orchestrator again to start an upgrade, and would run more tests again, then it would collect results. In this case, although the toolset can control the tests being run, and the order they are run, it has no real control of how the orchestrator deploys or upgrades the OpenStack deployment.\n\n- In this last use case, which is not part of this Spec currently, but is something that can be developed separate in the future, the process would be very similar to the second use case, with the one big difference of owning the orchestration part too. So in this case the toolset would have full control of not only the tests to be run but of the upgrade process too. \n\nNow, regarding separation of responsibilities,as mentioned in the Spec, this toolset could use (and probably will) existing projects like Rally (for example to run performance tests) and Tempest (for validating the health before and after an upgrade) along with other new tools developed to cover other areas (like the tool to determine at what level /release a service is). We didn\u0027t want to specify which of those existing  tools would be included in the toolset in the Spec, since we\u0027d like to make an assessment of their fit before defining that.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"b32dbcdda9504f774a6769925faea75cd3546d70","unresolved":false,"context_lines":[{"line_number":97,"context_line":"------------"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"**Functional**"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"},{"line_number":103,"context_line":"2. It must validate that services are actually at the correct release version at"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_0f36f851","line":100,"updated":"2017-03-30 06:28:35.000000000","message":"These numbered items are not rendered properly.\n\nhttp://docs-draft.openstack.org/95/449295/4/check/gate-qa-specs-docs-ubuntu-xenial/4fdbc2e//doc/build/html/specs/other/openstack-upgrade-testing.html#requirements","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"7d1aa1d6239df0980cbb7a5dc35daa1111273aab","unresolved":false,"context_lines":[{"line_number":97,"context_line":"------------"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"**Functional**"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"},{"line_number":103,"context_line":"2. It must validate that services are actually at the correct release version at"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_b9bbc5c2","line":100,"in_reply_to":"ffe62b97_0f36f851","updated":"2017-03-30 16:14:26.000000000","message":"You are right, I split the lines with line breaks to make them shorter to add readability while in review but that broke the format, I will remove the line breaks in the next patch.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"},{"line_number":103,"context_line":"2. It must validate that services are actually at the correct release version at"},{"line_number":104,"context_line":"any given time"},{"line_number":105,"context_line":"3. It must validate that all services are functional at any given time (e.g."},{"line_number":106,"context_line":"before, during and after an upgrade)"},{"line_number":107,"context_line":"4. It must provide a way of creating and validating persistent resources, like"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_0a580d54","line":104,"range":{"start_line":103,"start_character":0,"end_line":104,"end_character":14},"updated":"2017-03-30 16:49:49.000000000","message":"Is this the version of the code/binaries? I guess this depends very much on how the system is deployed and how the upgrade is performed.\nThings that grenade does today for instance is to upgrade, run DB migrations, and verify that a service can still work with the old configuration.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"1. The tool must be agnostic to the OpenStack environment and to the deployment"},{"line_number":102,"context_line":"tool used, performing actions consistently across different environments"},{"line_number":103,"context_line":"2. It must validate that services are actually at the correct release version at"},{"line_number":104,"context_line":"any given time"},{"line_number":105,"context_line":"3. It must validate that all services are functional at any given time (e.g."},{"line_number":106,"context_line":"before, during and after an upgrade)"},{"line_number":107,"context_line":"4. It must provide a way of creating and validating persistent resources, like"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_ed812c4c","line":104,"range":{"start_line":103,"start_character":0,"end_line":104,"end_character":14},"in_reply_to":"ffe62b97_0a580d54","updated":"2017-04-03 22:12:58.000000000","message":"Yes, we\u0027d like to figure out a way of determining what version the services are, so the tool can confirm an upgrade was in fact completed.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":110,"context_line":"period of time for any of the `supported services`_ listed below"},{"line_number":111,"context_line":"6. It must be capable to detect if all of the `supported services`_ listed below"},{"line_number":112,"context_line":"are fully available continuously during a specified period of time"},{"line_number":113,"context_line":"7. It must have a centralized store for logs of tests and data collected"},{"line_number":114,"context_line":"8. It must attempt to clean up after itself, if resources were created for"},{"line_number":115,"context_line":"testing or monitoring purposes they must be removed after they are no longer"},{"line_number":116,"context_line":"needed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_8582ceff","line":113,"range":{"start_line":113,"start_character":0,"end_line":113,"end_character":72},"updated":"2017-03-30 16:49:49.000000000","message":"I guess a folder is probably what you have in mind?","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":110,"context_line":"period of time for any of the `supported services`_ listed below"},{"line_number":111,"context_line":"6. It must be capable to detect if all of the `supported services`_ listed below"},{"line_number":112,"context_line":"are fully available continuously during a specified period of time"},{"line_number":113,"context_line":"7. It must have a centralized store for logs of tests and data collected"},{"line_number":114,"context_line":"8. It must attempt to clean up after itself, if resources were created for"},{"line_number":115,"context_line":"testing or monitoring purposes they must be removed after they are no longer"},{"line_number":116,"context_line":"needed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_0d10d000","line":113,"range":{"start_line":113,"start_character":0,"end_line":113,"end_character":72},"in_reply_to":"ffe62b97_8582ceff","updated":"2017-04-03 22:12:58.000000000","message":"We didn\u0027t want to include implementation details in the requirements, that is way we only mention a centralized store for data. This could be a DB, a file, etc. So it has not been defined yet, an analysis would have to be made to determine what is the best approach for this toolset.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":111,"context_line":"6. It must be capable to detect if all of the `supported services`_ listed below"},{"line_number":112,"context_line":"are fully available continuously during a specified period of time"},{"line_number":113,"context_line":"7. It must have a centralized store for logs of tests and data collected"},{"line_number":114,"context_line":"8. It must attempt to clean up after itself, if resources were created for"},{"line_number":115,"context_line":"testing or monitoring purposes they must be removed after they are no longer"},{"line_number":116,"context_line":"needed"},{"line_number":117,"context_line":"9. It must be pluggable in services meaning that when new services are ready to"},{"line_number":118,"context_line":"implement an upgrade strategy (for example zero downtime or zero impact), they"},{"line_number":119,"context_line":"can be easily added to the scope of the tool"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_e5a75aa7","line":116,"range":{"start_line":114,"start_character":0,"end_line":116,"end_character":6},"updated":"2017-03-30 16:49:49.000000000","message":"This is good to have, even though I doubt upgrade testing will be used very much on non-disposable environments, but I may be wrong.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":111,"context_line":"6. It must be capable to detect if all of the `supported services`_ listed below"},{"line_number":112,"context_line":"are fully available continuously during a specified period of time"},{"line_number":113,"context_line":"7. It must have a centralized store for logs of tests and data collected"},{"line_number":114,"context_line":"8. It must attempt to clean up after itself, if resources were created for"},{"line_number":115,"context_line":"testing or monitoring purposes they must be removed after they are no longer"},{"line_number":116,"context_line":"needed"},{"line_number":117,"context_line":"9. It must be pluggable in services meaning that when new services are ready to"},{"line_number":118,"context_line":"implement an upgrade strategy (for example zero downtime or zero impact), they"},{"line_number":119,"context_line":"can be easily added to the scope of the tool"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_6dfe3caf","line":116,"range":{"start_line":114,"start_character":0,"end_line":116,"end_character":6},"in_reply_to":"ffe62b97_e5a75aa7","updated":"2017-04-03 22:12:58.000000000","message":"Agree","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":122,"context_line":"11. It must provide a common public interface that the toolset can use to"},{"line_number":123,"context_line":"communicate with deployment tools so certain steps of the deployment or the"},{"line_number":124,"context_line":"upgrade can be triggered"},{"line_number":125,"context_line":"12. In case of cascade errors the tool must stop actions like creating resources"},{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_0520fef6","line":127,"range":{"start_line":125,"start_character":0,"end_line":127,"end_character":11},"updated":"2017-03-30 16:49:49.000000000","message":"Roll back is a very hard requirement, and I don\u0027t think it should be in the initial set.\nIt\u0027s fine to call it out as an eventual target, but I think you should identify an MVP as masayuki wrote, and focus on that first.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":122,"context_line":"11. It must provide a common public interface that the toolset can use to"},{"line_number":123,"context_line":"communicate with deployment tools so certain steps of the deployment or the"},{"line_number":124,"context_line":"upgrade can be triggered"},{"line_number":125,"context_line":"12. In case of cascade errors the tool must stop actions like creating resources"},{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_8d9f405e","line":127,"range":{"start_line":125,"start_character":0,"end_line":127,"end_character":11},"in_reply_to":"ffe62b97_0520fef6","updated":"2017-04-03 22:12:58.000000000","message":"This is not intended to be a rollback. Since the process to test an upgrade could be very large, this requirement is attempting to say that if for some reason, several errors start occurring during the process (meaning something broke badly), the toolset should stop the process, clean up and report the issues instead of carrying forward which would lead to have lots and lots of errors therefore difficult to debug. I\u0027ll try to re phrase this point since it is obviously not clear enough.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":125,"context_line":"12. In case of cascade errors the tool must stop actions like creating resources"},{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_45d966f2","line":128,"range":{"start_line":128,"start_character":0,"end_line":128,"end_character":55},"updated":"2017-03-30 16:49:49.000000000","message":"I understood testing (beyond availability of services) is was going to out of scope for this. Tempest Rally etc can be used for testing?","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":125,"context_line":"12. In case of cascade errors the tool must stop actions like creating resources"},{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_ed380c3e","line":128,"range":{"start_line":128,"start_character":0,"end_line":128,"end_character":55},"in_reply_to":"ffe62b97_45d966f2","updated":"2017-04-03 22:12:58.000000000","message":"The toolset will run different types of test in an attempt to provide feedback from an upgrade process, some of these tests could be through Tempest, and some could be developed with the toolset. An example of this is something we call \"Persist Resources Tests\", which are tests that create resources before an upgrade (like VMs, Containers, Volumes, etc.) and validate them during and after an upgrade to make sure they were not corrupted during the upgrade. Tempest doesn\u0027t include tests that cover this area, resources created with Tempest get deleted at the end of the tests, so the toolset would need to have an implementation of these tests. This requirement is saying that all these tests should not use admin users.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"},{"line_number":132,"context_line":"16. It should verify that all requests made during an upgrade are honored at"},{"line_number":133,"context_line":"some point successfully, validating that they are not just added to a queue but"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_85b64e8a","line":130,"range":{"start_line":129,"start_character":0,"end_line":130,"end_character":68},"updated":"2017-03-30 16:49:49.000000000","message":"This is a very open definition, and not something that should be in scope here I think. You can have performance measurement running through the upgrade process with some other tool.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":126,"context_line":"that would lead to an even more unstable environment, attempt to clean up and"},{"line_number":127,"context_line":"report back"},{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"},{"line_number":132,"context_line":"16. It should verify that all requests made during an upgrade are honored at"},{"line_number":133,"context_line":"some point successfully, validating that they are not just added to a queue but"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_6d72bc13","line":130,"range":{"start_line":129,"start_character":0,"end_line":130,"end_character":68},"in_reply_to":"ffe62b97_85b64e8a","updated":"2017-04-03 22:12:58.000000000","message":"In order for the toolset to be able to provide feedback for the \"Zero impact upgrade\", the toolset must be capable of measuring the performance before, during and after an upgrade. This requirement is to reflect that need. Again, the toolset may use Rally to perform this if make sense, if it doesn\u0027t it would implement a module to do it.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"},{"line_number":132,"context_line":"16. It should verify that all requests made during an upgrade are honored at"},{"line_number":133,"context_line":"some point successfully, validating that they are not just added to a queue but"},{"line_number":134,"context_line":"are actually processed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_c518d685","line":131,"range":{"start_line":131,"start_character":0,"end_line":131,"end_character":77},"updated":"2017-03-30 16:49:49.000000000","message":"I think it should get an input from the deployment system about what\u0027s expected to be there and act accordingly.\nI\u0027m not sure discovery is the best option. If we run upgrade on top of an existing system it may be that something is broken, and not running. Discovery might then skip something rather than reporting an issue.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":128,"context_line":"13. It must run tests using non-admin OpenStack user(s)"},{"line_number":129,"context_line":"14. It should be capable of measuring the performance of the"},{"line_number":130,"context_line":"`supported services`_ listed below during a specified period of time"},{"line_number":131,"context_line":"15. It should auto discover cloud services and configure the tool accordingly"},{"line_number":132,"context_line":"16. It should verify that all requests made during an upgrade are honored at"},{"line_number":133,"context_line":"some point successfully, validating that they are not just added to a queue but"},{"line_number":134,"context_line":"are actually processed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_6d4bdcaf","line":131,"range":{"start_line":131,"start_character":0,"end_line":131,"end_character":77},"in_reply_to":"ffe62b97_c518d685","updated":"2017-04-03 22:12:58.000000000","message":"Yes, we have debated about this point many times. We decided to include it as it is just to simplify the configuration of the tool for the end user, but we will reconsider this point. I\u0027ll add an update of this in the new patchset.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":137,"context_line":"reuse of test resources from existing OpenStack test tools"},{"line_number":138,"context_line":"19. It should be capable of performing tests on one of the `supported services`_"},{"line_number":139,"context_line":"at a time"},{"line_number":140,"context_line":"20. It could include a GUI where results can be easily interpreted and could"},{"line_number":141,"context_line":"include trends"},{"line_number":142,"context_line":"21. It could be capable of verifying if the data plane is accessible during a"},{"line_number":143,"context_line":"specified period of time"},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_570ebd32","line":141,"range":{"start_line":140,"start_character":0,"end_line":141,"end_character":14},"updated":"2017-03-30 16:49:49.000000000","message":"I think the best requirement to have here is to collect results in some kind of machine readable format / DB. So you can eventually build a GUI on top.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":137,"context_line":"reuse of test resources from existing OpenStack test tools"},{"line_number":138,"context_line":"19. It should be capable of performing tests on one of the `supported services`_"},{"line_number":139,"context_line":"at a time"},{"line_number":140,"context_line":"20. It could include a GUI where results can be easily interpreted and could"},{"line_number":141,"context_line":"include trends"},{"line_number":142,"context_line":"21. It could be capable of verifying if the data plane is accessible during a"},{"line_number":143,"context_line":"specified period of time"},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_6d7cfcb4","line":141,"range":{"start_line":140,"start_character":0,"end_line":141,"end_character":14},"in_reply_to":"ffe62b97_570ebd32","updated":"2017-04-03 22:12:58.000000000","message":"Requirements with the word \"could\" are more like nice to have than critical requirements. The toolset will be running different kind of tests that could benefit from showing a graph to be clearer, for example API uptime tests, or some performance tests. This requirement is to show the low priority of including some type of visualization for these results. But again, it could be as simple as a matplotlib graph and as of now it is just a nice to have.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"b32dbcdda9504f774a6769925faea75cd3546d70","unresolved":false,"context_lines":[{"line_number":140,"context_line":"20. It could include a GUI where results can be easily interpreted and could"},{"line_number":141,"context_line":"include trends"},{"line_number":142,"context_line":"21. It could be capable of verifying if the data plane is accessible during a"},{"line_number":143,"context_line":"specified period of time"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"**Non-Functional**"},{"line_number":146,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_ef4074a7","line":143,"updated":"2017-03-30 06:28:35.000000000","message":"I feel these are too much functionalities... So, I\u0027d like to know the MVP(Minimum Viable Product) in this spec.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"7d1aa1d6239df0980cbb7a5dc35daa1111273aab","unresolved":false,"context_lines":[{"line_number":140,"context_line":"20. It could include a GUI where results can be easily interpreted and could"},{"line_number":141,"context_line":"include trends"},{"line_number":142,"context_line":"21. It could be capable of verifying if the data plane is accessible during a"},{"line_number":143,"context_line":"specified period of time"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"**Non-Functional**"},{"line_number":146,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_3968d51f","line":143,"in_reply_to":"ffe62b97_ef4074a7","updated":"2017-03-30 16:14:26.000000000","message":"The short answer is that every requirement with \"must\" or \"should\" needs to be part of the MVP.\nThe requirements were prioritized using the MoSCoW method:\nhttps://en.wikipedia.org/wiki/MoSCoW_method\n\nWe plan to start implementing the modules that are mentioned in the milestones section, which would allow us to test rolling upgrades. And then start implementing the other modules in future release(s) that would allow us to test zero downtime and zero impact.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"b32dbcdda9504f774a6769925faea75cd3546d70","unresolved":false,"context_lines":[{"line_number":177,"context_line":"- jlwhite"},{"line_number":178,"context_line":"- kvcobb"},{"line_number":179,"context_line":"- dankolbrs"},{"line_number":180,"context_line":"- scarab_"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"Work Items"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_cf3b7035","line":180,"updated":"2017-03-30 06:28:35.000000000","message":"heh, this is rendered as a link","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"7d1aa1d6239df0980cbb7a5dc35daa1111273aab","unresolved":false,"context_lines":[{"line_number":177,"context_line":"- jlwhite"},{"line_number":178,"context_line":"- kvcobb"},{"line_number":179,"context_line":"- dankolbrs"},{"line_number":180,"context_line":"- scarab_"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"Work Items"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_f9f82d4a","line":180,"in_reply_to":"ffe62b97_cf3b7035","updated":"2017-03-30 16:14:26.000000000","message":"yep, I will scape the underscore so it shows correctly in the next patch.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"bb4e313139042a850d95655a4bbf800da6a76ad0","unresolved":false,"context_lines":[{"line_number":215,"context_line":"- Release Version Validator"},{"line_number":216,"context_line":"- Report Generator (very basic version)"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"Target milestone for completion:"},{"line_number":219,"context_line":"  Pike-3"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Diagram"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ffe62b97_2a519163","line":219,"range":{"start_line":218,"start_character":0,"end_line":219,"end_character":8},"updated":"2017-03-30 16:49:49.000000000","message":"This sounds optimistic :)","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":9152,"name":"Castulo J. Martinez","email":"castulo.martinez@intel.com","username":"cas"},"change_message_id":"1805b87f92eb7b585f5125d911474f3c1b432384","unresolved":false,"context_lines":[{"line_number":215,"context_line":"- Release Version Validator"},{"line_number":216,"context_line":"- Report Generator (very basic version)"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"Target milestone for completion:"},{"line_number":219,"context_line":"  Pike-3"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Diagram"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dfeb2761_8d5aa04c","line":219,"range":{"start_line":218,"start_character":0,"end_line":219,"end_character":8},"in_reply_to":"ffe62b97_2a519163","updated":"2017-04-03 22:12:58.000000000","message":"hehe, it probably is.","commit_id":"0778d59e981d878781b37512cf159165f80b08a9"},{"author":{"_account_id":4340,"name":"Sulochan Acharya","email":"sulochan@gmail.com","username":"sulochan-acharya"},"change_message_id":"4c6a60555829b9d1bf550c8d31ef695a2f169186","unresolved":false,"context_lines":[{"line_number":12,"context_line":"As the OpenStack community works to improve the upgrade experience, there is a"},{"line_number":13,"context_line":"growing need to provide a variety of test tools which help to exercise,"},{"line_number":14,"context_line":"harness and validate the upgrade approaches defined by the community."},{"line_number":15,"context_line":"This spec describes the creation of a new toolset that could be used to test"},{"line_number":16,"context_line":"upgrades of OpenStack services."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Problem Description"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bff0334d_eb80dac1","line":16,"range":{"start_line":15,"start_character":0,"end_line":16,"end_character":31},"updated":"2017-04-10 07:59:26.000000000","message":"The scope of the tool needs some clarification, are we testing by spinning up a new env with a given version either devstack, OSA etc or are we proposing that this tool will work on a live production environment as well with some --dry-run type capability. I think it will help to include that here.","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"},{"author":{"_account_id":23741,"name":"Dan Kolb","email":"openstack@dankolb.net","username":"dankolbrs"},"change_message_id":"c16abba35f29eeabc27e87f07c6b632c58a58a86","unresolved":false,"context_lines":[{"line_number":33,"context_line":"- `supports-zero-impact-upgrade`_"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Currently there is only one project in the OpenStack ecosystem that can be used"},{"line_number":36,"context_line":"to perform some level of upgrade testing, this project is `Grenade`_, and it was"},{"line_number":37,"context_line":"designed with the main purpose of testing backward compatibility of project"},{"line_number":38,"context_line":"configuration, so it does not provide tests or tools for validating the"},{"line_number":39,"context_line":"different maturity levels of an upgrade. In order for projects to self assess"}],"source_content_type":"text/x-rst","patch_set":6,"id":"dfeb2761_e52a402a","line":36,"updated":"2017-04-04 19:13:19.000000000","message":"Minor nitpick here, this sentence is super long and may be better broken into two.\n\n...perform some level of upgrade testing, that project being Grenade. Grenade was designed with the main purpose...","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"},{"author":{"_account_id":4340,"name":"Sulochan Acharya","email":"sulochan@gmail.com","username":"sulochan-acharya"},"change_message_id":"4c6a60555829b9d1bf550c8d31ef695a2f169186","unresolved":false,"context_lines":[{"line_number":55,"context_line":"be used to assess if an upgrade complies with the requirements for being"},{"line_number":56,"context_line":"recognized as a `rolling upgrade`_, a `zero downtime upgrade`_ or a"},{"line_number":57,"context_line":"`zero impact upgrade`_."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The toolset will provide the necessary tools for testing the behavior of an"},{"line_number":60,"context_line":"upgrade but will not orchestrate the upgrade process itself initially, this"},{"line_number":61,"context_line":"responsibility will belong to the deployer, making this tool agnostic to the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bff0334d_eb38daee","line":58,"updated":"2017-04-10 07:59:26.000000000","message":"I think it will help to say how we are doing this, we have some mentions below but it will clarify what the tool is about if we specifically say we will at least have - API driven service, a UI for user control ?","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"},{"author":{"_account_id":4340,"name":"Sulochan Acharya","email":"sulochan@gmail.com","username":"sulochan-acharya"},"change_message_id":"cc787febbdb3b003434db9ab3856745812bac68b","unresolved":false,"context_lines":[{"line_number":79,"context_line":"This toolset will be able to be used in CI systems as well as in a computer"},{"line_number":80,"context_line":"locally, so developers will be able to leverage it in a development environment."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Use Cases"},{"line_number":83,"context_line":"---------"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"There are three main use cases for the toolset:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bff0334d_e6548f00","line":82,"range":{"start_line":82,"start_character":0,"end_line":82,"end_character":9},"updated":"2017-04-10 08:11:46.000000000","message":"I think the use cases are actually too broad. I understand that perhaps for the purpose of this spec we leave it as such, but each of these could be broken down into multiple user stories with milestones attached to it.","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"},{"author":{"_account_id":4340,"name":"Sulochan Acharya","email":"sulochan@gmail.com","username":"sulochan-acharya"},"change_message_id":"cc787febbdb3b003434db9ab3856745812bac68b","unresolved":false,"context_lines":[{"line_number":94,"context_line":"  to test upgrades, the consumer would be responsible for deploying the"},{"line_number":95,"context_line":"  environment, running the validators and observers in the correct place in the"},{"line_number":96,"context_line":"  upgrade process, performing the upgrade, and stopping the observers once the"},{"line_number":97,"context_line":"  upgrade finishes."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"- The second use case is about the toolset having partial control over the"},{"line_number":100,"context_line":"  deployment/upgrade process but full control of the test process. In this use"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bff0334d_26d4b747","line":97,"updated":"2017-04-10 08:11:46.000000000","message":"so I understand this as the following: \nOpenstack operator will run this tool to observe how an external upgrade is working. The operator will do the upgrading, but running this tool at the same time to record, availability, performance etc so we get a good picture of how the upgrade went ? is this understanding correct ? I","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"},{"author":{"_account_id":4340,"name":"Sulochan Acharya","email":"sulochan@gmail.com","username":"sulochan-acharya"},"change_message_id":"cc787febbdb3b003434db9ab3856745812bac68b","unresolved":false,"context_lines":[{"line_number":111,"context_line":"  orchestrator is to be used, a new driver will have to be implemented for this"},{"line_number":112,"context_line":"  orchestrator."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"- The third use case (**Not in scope for this Spec**) is about having full"},{"line_number":115,"context_line":"  control over the deployment/upgrade process and full control of the test"},{"line_number":116,"context_line":"  process. This use case is very similar to the second one where the toolset"},{"line_number":117,"context_line":"  signals the orchestrator to initiate a deployment/upgrade, but the one big"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bff0334d_86c4a392","line":114,"range":{"start_line":114,"start_character":0,"end_line":114,"end_character":74},"updated":"2017-04-10 08:11:46.000000000","message":"Not sure i understand this, if its not in scope for this spec, why is it here ?","commit_id":"d59d35d3face2ed259e5615c1616d9dea1d95955"}]}