)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"3740bd3420880ca437a36b34a4b0676c71de78e2","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Tests that target and API which does not provide a version discovery"},{"line_number":10,"context_line":"mechanism, and which introduces backward incompatibilites cannot"},{"line_number":11,"context_line":"leave in Tempest due to its branchless nature."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"With this commit I propose to extend the list of exceptions from"},{"line_number":14,"context_line":"the standard removal procedure to include this group of tests."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"da36d5c6_b5a355ff","line":11,"updated":"2017-02-23 13:21:41.000000000","message":"live?","commit_id":"acad55aa358c72d09566034d4207d7d4d3d9d33b"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"3740bd3420880ca437a36b34a4b0676c71de78e2","unresolved":false,"context_lines":[{"line_number":13,"context_line":"With this commit I propose to extend the list of exceptions from"},{"line_number":14,"context_line":"the standard removal procedure to include this group of tests."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Example: https://review.openstack.org/#q,I84626976ca729f65b0ee0ea6afe5c9a6a408eecc,n,z"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: I209b0b3d470bd284949a77d9c266d78e7c5a5c13"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"da36d5c6_d5a8c9dd","line":16,"updated":"2017-02-23 13:21:41.000000000","message":"I think this should be just a Change-Id not gerrit url. Otherwise, the gerrit rendering is something wrong.","commit_id":"acad55aa358c72d09566034d4207d7d4d3d9d33b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"e4be1efcadc2a925a6c4cc0496a08ec18ca0dd35","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Tests that target and API which does not provide a version discovery"},{"line_number":10,"context_line":"mechanism, and which introduces backward incompatibilites cannot"},{"line_number":11,"context_line":"leave in Tempest due to its branchless nature."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"With this commit I propose to extend the list of exceptions from"},{"line_number":14,"context_line":"the standard removal procedure to include this group of tests."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"da36d5c6_09aec580","line":11,"range":{"start_line":11,"start_character":0,"end_line":11,"end_character":5},"updated":"2017-02-23 16:18:07.000000000","message":"as Masayuki commented, -\u003elive","commit_id":"cbb086af97b3ad71f5a38bf9f4eb883790ff36da"}],"doc/source/test-removal.rst":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"79d19ad73109b0c82e5ccfed4be1805c3f207ace","unresolved":false,"context_lines":[{"line_number":133,"context_line":" #. A revert for a patch which added a broken test, or testing which didn\u0027t"},{"line_number":134,"context_line":"    actually run in the gate (basically any revert for something which"},{"line_number":135,"context_line":"    shouldn\u0027t have been added)"},{"line_number":136,"context_line":" #. The test targets and API which includes a non-discoverable, backward"},{"line_number":137,"context_line":"    incompatible change in one of the releases supported by Tempest or in"},{"line_number":138,"context_line":"    the current version under development."},{"line_number":139,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":140,"context_line":"    Tempest. If the version of the API is not discoverable, a test have no"},{"line_number":141,"context_line":"    way to enforce the correct behavior, so it must be removed."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":144,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da36d5c6_35b12448","line":141,"range":{"start_line":136,"start_character":0,"end_line":141,"end_character":63},"updated":"2017-02-23 13:49:41.000000000","message":"Thanks, this is nice and totally agree but removal of these kind of tests should fall under 3rd prong.\n\nSo we should take care of 3rd prong for those removal ?","commit_id":"acad55aa358c72d09566034d4207d7d4d3d9d33b"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"e4be1efcadc2a925a6c4cc0496a08ec18ca0dd35","unresolved":false,"context_lines":[{"line_number":38,"context_line":" #. The test proposed for removal has a failure rate \u003c  0.50% in the gate over"},{"line_number":39,"context_line":"    the past release (the value and interval will likely be adjusted in the"},{"line_number":40,"context_line":"    future)"},{"line_number":41,"context_line":" #. .. _`prong #3`: There must not be an external user/consumer of tempest"},{"line_number":42,"context_line":"    that depends on the test proposed for removal"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"The answers to 1 and 2 are easy to verify. For 1 just provide a link to the new"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da36d5c6_4a418ba8","line":41,"range":{"start_line":41,"start_character":7,"end_line":41,"end_character":18},"updated":"2017-02-23 16:18:07.000000000","message":"it should be _prong #3:","commit_id":"cbb086af97b3ad71f5a38bf9f4eb883790ff36da"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"e4be1efcadc2a925a6c4cc0496a08ec18ca0dd35","unresolved":false,"context_lines":[{"line_number":136,"context_line":" #. Tests that would become out of scope as a consequence of an API change,"},{"line_number":137,"context_line":"    as described in `API Compatibility`_."},{"line_number":138,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":139,"context_line":"    Tempest. Such test must still honor `prong #3`_."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":142,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da36d5c6_8aab8374","line":139,"range":{"start_line":139,"start_character":39,"end_line":139,"end_character":52},"updated":"2017-02-23 16:18:07.000000000","message":"need to fix this link.","commit_id":"cbb086af97b3ad71f5a38bf9f4eb883790ff36da"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"2973ca3cc6a72e936690730769196d035d3b5f11","unresolved":false,"context_lines":[{"line_number":135,"context_line":" #. A revert for a patch which added a broken test, or testing which didn\u0027t"},{"line_number":136,"context_line":"    actually run in the gate (basically any revert for something which"},{"line_number":137,"context_line":"    shouldn\u0027t have been added)"},{"line_number":138,"context_line":" #. Tests that would become out of scope as a consequence of an API change,"},{"line_number":139,"context_line":"    as described in `API Compatibility`_."},{"line_number":140,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":141,"context_line":"    Tempest. Such test must still honor `prong #3`_."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":144,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_2328fde1","line":141,"range":{"start_line":138,"start_character":0,"end_line":141,"end_character":52},"updated":"2017-02-23 19:39:11.000000000","message":"we should have this rule for API tests only, making this rule on scenario tests also can impact scenario testing coverage. I mean any scenario tests using such API as one of the service to do some operation should be ok.","commit_id":"b200555709f12bb3b0e5c6cf38f10e752c7f7977"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"1c6864d1f69944e5777ced5cd4d8c6c1f7d15255","unresolved":false,"context_lines":[{"line_number":135,"context_line":" #. A revert for a patch which added a broken test, or testing which didn\u0027t"},{"line_number":136,"context_line":"    actually run in the gate (basically any revert for something which"},{"line_number":137,"context_line":"    shouldn\u0027t have been added)"},{"line_number":138,"context_line":" #. Tests that would become out of scope as a consequence of an API change,"},{"line_number":139,"context_line":"    as described in `API Compatibility`_."},{"line_number":140,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":141,"context_line":"    Tempest. Such test must still honor `prong #3`_."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":144,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_a67b1a27","line":141,"range":{"start_line":138,"start_character":0,"end_line":141,"end_character":52},"in_reply_to":"da36d5c6_069dc6bb","updated":"2017-02-23 20:14:19.000000000","message":"got it. that make sense. API change in backward incompatible manner can affect scenario testing too so i agree to remove such API testing.","commit_id":"b200555709f12bb3b0e5c6cf38f10e752c7f7977"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"d9662f031db825a4f7a655ff59c4662595c6ae00","unresolved":false,"context_lines":[{"line_number":135,"context_line":" #. A revert for a patch which added a broken test, or testing which didn\u0027t"},{"line_number":136,"context_line":"    actually run in the gate (basically any revert for something which"},{"line_number":137,"context_line":"    shouldn\u0027t have been added)"},{"line_number":138,"context_line":" #. Tests that would become out of scope as a consequence of an API change,"},{"line_number":139,"context_line":"    as described in `API Compatibility`_."},{"line_number":140,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":141,"context_line":"    Tempest. Such test must still honor `prong #3`_."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":144,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_034cd89c","line":141,"range":{"start_line":138,"start_character":0,"end_line":141,"end_character":52},"in_reply_to":"da36d5c6_2328fde1","updated":"2017-02-23 20:07:24.000000000","message":"I don\u0027t think we need to differentiate between scenario and api here. While api tests are concentrating on api validation the same issues which affect us on api jobs will also be an issue for scenario tests.","commit_id":"b200555709f12bb3b0e5c6cf38f10e752c7f7977"},{"author":{"_account_id":6167,"name":"Ken\u0027ichi Ohmichi","email":"ken1ohmichi@gmail.com","username":"oomichi"},"change_message_id":"4c4502bcde5e7b2e47e6d61cfde9b58aace624ae","unresolved":false,"context_lines":[{"line_number":135,"context_line":" #. A revert for a patch which added a broken test, or testing which didn\u0027t"},{"line_number":136,"context_line":"    actually run in the gate (basically any revert for something which"},{"line_number":137,"context_line":"    shouldn\u0027t have been added)"},{"line_number":138,"context_line":" #. Tests that would become out of scope as a consequence of an API change,"},{"line_number":139,"context_line":"    as described in `API Compatibility`_."},{"line_number":140,"context_line":"    Such tests cannot live in Tempest because of the branchless nature of"},{"line_number":141,"context_line":"    Tempest. Such test must still honor `prong #3`_."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"For the first exception type the only types of testing in tree which have been"},{"line_number":144,"context_line":"declared out of scope at this point are:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_069dc6bb","line":141,"range":{"start_line":138,"start_character":0,"end_line":141,"end_character":52},"in_reply_to":"da36d5c6_2328fde1","updated":"2017-02-23 20:07:25.000000000","message":"I think scenario tests are based on API tests. That means Tempest verifies API behaviors on each API test as the basic step and verifies combination behaviors on scenario tests after API tests. Then if Tempest doesn\u0027t verify the API behaviors on some APIs, I feel Tempest doesn\u0027t need to implement the scenario tests.","commit_id":"b200555709f12bb3b0e5c6cf38f10e752c7f7977"}]}