)]}'
{"specs/kilo/tests-requirements-and-goals.rst":[{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"d9c6769d9a9c489e2d82049ea24c949fe78acf7a","unresolved":false,"context_lines":[{"line_number":51,"context_line":"- A test to detect changes in dependency. Should call a reasonable amount of"},{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_2de21830","line":54,"updated":"2015-05-27 21:34:48.000000000","message":"This kind of thing gets caught naturally in our tests, I don\u0027t think a special test is needed for it.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":6554,"name":"Spencer Krum","email":"nibz@spencerkrum.com","username":"krum-spencer"},"change_message_id":"4a27b874e363376efedd5e428232b44fb9a35074","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":58,"context_line":"  Action on detect: Fail"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_e061fea9","line":55,"updated":"2015-05-25 19:11:41.000000000","message":"Why three times?","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":5241,"name":"Martin Magr","email":"mmagr@redhat.com","username":"mmagr"},"change_message_id":"3eb38d8ac1f9c639ad2bc7694137efd8aee94398","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":58,"context_line":"  Action on detect: Fail"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5afe65bd_ef6f9190","line":55,"in_reply_to":"7a016987_3197e749","updated":"2015-06-02 11:46:57.000000000","message":"+1 for just two runs.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"da3c578510194b612b5efc1df18c80804df0b763","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":58,"context_line":"  Action on detect: Fail"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_3197e749","line":55,"in_reply_to":"7a016987_4d1d5465","updated":"2015-05-28 21:18:39.000000000","message":"+1 for 2","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"d9c6769d9a9c489e2d82049ea24c949fe78acf7a","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":58,"context_line":"  Action on detect: Fail"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_4d1d5465","line":55,"in_reply_to":"7a016987_aa474b4a","updated":"2015-05-27 21:34:48.000000000","message":"It should only be two times. If changes happen on the second time that is a failure. This is how the puppetlabs modules are tested.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"b86ecd25f9b93c7b8e99c031d1a2324e3ae48df1","unresolved":false,"context_lines":[{"line_number":52,"context_line":"  classes that use dependencies as much as possible."},{"line_number":53,"context_line":"  Test group: integration, beaker-rspec"},{"line_number":54,"context_line":"  Action on detect: Fail"},{"line_number":55,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":56,"context_line":"  three times and check for errors."},{"line_number":57,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":58,"context_line":"  Action on detect: Fail"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_aa474b4a","line":55,"in_reply_to":"7a016987_e061fea9","updated":"2015-05-26 20:18:12.000000000","message":"No specific reason, I usually think that odd number is better. If I\u0027m getting a idempotency problem while applying manually I\u0027d retry a third time anyway","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":6554,"name":"Spencer Krum","email":"nibz@spencerkrum.com","username":"krum-spencer"},"change_message_id":"4a27b874e363376efedd5e428232b44fb9a35074","unresolved":false,"context_lines":[{"line_number":64,"context_line":"  Code reviewers should be particularly alert to changes on test sets"},{"line_number":65,"context_line":"  for a parameter change, as they suggest backwards incompatibility."},{"line_number":66,"context_line":"  Test group: functional, beaker-rspec, with hacks"},{"line_number":67,"context_line":"  Action on detect: Add deprecation warnings"},{"line_number":68,"context_line":"- To detect parameters inconsistency in ini files usage by manifests and"},{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_c05602ce","line":67,"updated":"2015-05-25 19:11:41.000000000","message":"This sounds like a very long test to run, and complicated to implement. Maybe move it to secondary priority?","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":5241,"name":"Martin Magr","email":"mmagr@redhat.com","username":"mmagr"},"change_message_id":"3eb38d8ac1f9c639ad2bc7694137efd8aee94398","unresolved":false,"context_lines":[{"line_number":64,"context_line":"  Code reviewers should be particularly alert to changes on test sets"},{"line_number":65,"context_line":"  for a parameter change, as they suggest backwards incompatibility."},{"line_number":66,"context_line":"  Test group: functional, beaker-rspec, with hacks"},{"line_number":67,"context_line":"  Action on detect: Add deprecation warnings"},{"line_number":68,"context_line":"- To detect parameters inconsistency in ini files usage by manifests and"},{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5afe65bd_afc19930","line":67,"in_reply_to":"7a016987_11a2ab99","updated":"2015-06-02 11:46:57.000000000","message":"Do you have an idea how we can achieve such testing?","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"da3c578510194b612b5efc1df18c80804df0b763","unresolved":false,"context_lines":[{"line_number":64,"context_line":"  Code reviewers should be particularly alert to changes on test sets"},{"line_number":65,"context_line":"  for a parameter change, as they suggest backwards incompatibility."},{"line_number":66,"context_line":"  Test group: functional, beaker-rspec, with hacks"},{"line_number":67,"context_line":"  Action on detect: Add deprecation warnings"},{"line_number":68,"context_line":"- To detect parameters inconsistency in ini files usage by manifests and"},{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_11a2ab99","line":67,"in_reply_to":"7a016987_6a2a68e9","updated":"2015-05-28 21:18:39.000000000","message":"agreed, I think though we\u0027ve made these non-voting right?","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"b86ecd25f9b93c7b8e99c031d1a2324e3ae48df1","unresolved":false,"context_lines":[{"line_number":64,"context_line":"  Code reviewers should be particularly alert to changes on test sets"},{"line_number":65,"context_line":"  for a parameter change, as they suggest backwards incompatibility."},{"line_number":66,"context_line":"  Test group: functional, beaker-rspec, with hacks"},{"line_number":67,"context_line":"  Action on detect: Add deprecation warnings"},{"line_number":68,"context_line":"- To detect parameters inconsistency in ini files usage by manifests and"},{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_6a2a68e9","line":67,"in_reply_to":"7a016987_c05602ce","updated":"2015-05-26 20:18:12.000000000","message":"This is one of the most important tests, since changing parameters defaults and parameters behaviour is one of the main causes of disruption among downstream composition layers. I can\u0027t think of any way to lower its impact, even if its role is more to create warnings and not to really stop the patch in case of failure","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":6554,"name":"Spencer Krum","email":"nibz@spencerkrum.com","username":"krum-spencer"},"change_message_id":"4a27b874e363376efedd5e428232b44fb9a35074","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":71,"context_line":"  Action on detect: Fail"},{"line_number":72,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":73,"context_line":"  a scope, then track all the modifications done (from catalog log), and"},{"line_number":74,"context_line":"  notify all unrecognized ones."},{"line_number":75,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_00041ac4","line":72,"updated":"2015-05-25 19:11:41.000000000","message":"It\u0027d be interesting to see what resources are instantiatied by what moudules. Compile the catalog with --graph and inspect the resulting output/datastructure.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"b86ecd25f9b93c7b8e99c031d1a2324e3ae48df1","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  provider, it should be sufficient to run functional test."},{"line_number":70,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":71,"context_line":"  Action on detect: Fail"},{"line_number":72,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":73,"context_line":"  a scope, then track all the modifications done (from catalog log), and"},{"line_number":74,"context_line":"  notify all unrecognized ones."},{"line_number":75,"context_line":"  Test group: functional, beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_cde62144","line":72,"in_reply_to":"7a016987_00041ac4","updated":"2015-05-26 20:18:12.000000000","message":"I\u0027ll add this suggestion to test description","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"d9c6769d9a9c489e2d82049ea24c949fe78acf7a","unresolved":false,"context_lines":[{"line_number":73,"context_line":"  a scope, then track all the modifications done (from catalog log), and"},{"line_number":74,"context_line":"  notify all unrecognized ones."},{"line_number":75,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":76,"context_line":"  Action on detect: ??"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Every significant new features should add new functional tests to check"},{"line_number":79,"context_line":"for recurring pattern in the above scenarios, in particular at least:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_2d50d8d3","line":76,"updated":"2015-05-27 21:34:48.000000000","message":"I don\u0027t think we should attempt to test this automatically, it is purely up to the code reviewers to determine what is out of scope for a patch.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"da3c578510194b612b5efc1df18c80804df0b763","unresolved":false,"context_lines":[{"line_number":73,"context_line":"  a scope, then track all the modifications done (from catalog log), and"},{"line_number":74,"context_line":"  notify all unrecognized ones."},{"line_number":75,"context_line":"  Test group: functional, beaker-rspec"},{"line_number":76,"context_line":"  Action on detect: ??"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Every significant new features should add new functional tests to check"},{"line_number":79,"context_line":"for recurring pattern in the above scenarios, in particular at least:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_b1b757d7","line":76,"in_reply_to":"7a016987_2d50d8d3","updated":"2015-05-28 21:18:39.000000000","message":"I\u0027d love to suggest our catalog diff tool if we could agree on a basic set of nodes that we\u0027d run diffs against.","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"d9c6769d9a9c489e2d82049ea24c949fe78acf7a","unresolved":false,"context_lines":[{"line_number":80,"context_line":"- catalog compilation"},{"line_number":81,"context_line":"- idempotency"},{"line_number":82,"context_line":"- desired effects (the right port is open, service responds to a list"},{"line_number":83,"context_line":"command, etc)"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Minor changes should only warrant unit test changes"},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_ed6080a6","line":83,"updated":"2015-05-27 21:34:48.000000000","message":"++","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"d9c6769d9a9c489e2d82049ea24c949fe78acf7a","unresolved":false,"context_lines":[{"line_number":82,"context_line":"- desired effects (the right port is open, service responds to a list"},{"line_number":83,"context_line":"command, etc)"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Minor changes should only warrant unit test changes"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7a016987_cd5704c9","line":85,"updated":"2015-05-27 21:34:48.000000000","message":"++","commit_id":"b3dcb0c70b8a1dea971f276a420001d8c5ad2e9e"}],"specs/tests-requirements-and-goals.rst":[{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"a14adda1841c1264481e0a123c0036aa4012cab3","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"A brief survey in downstream, showed some recurring pattern when dealing with"},{"line_number":20,"context_line":"bugs from puppet modules:"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"- a change in a common dependency (usually concat)"},{"line_number":23,"context_line":"- idempotency not respected"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_1d33b315","line":20,"updated":"2015-05-05 20:30:21.000000000","message":"Is this survey publicly available? Could you include it in the references?","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"A brief survey in downstream, showed some recurring pattern when dealing with"},{"line_number":20,"context_line":"bugs from puppet modules:"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"- a change in a common dependency (usually concat)"},{"line_number":23,"context_line":"- idempotency not respected"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_7717eca3","line":20,"in_reply_to":"3acd31a7_1d33b315","updated":"2015-05-06 17:45:42.000000000","message":"The survey was not conducted with a statistically significant sample, and there is no official document for it. If anyone could to organize something more scientific than what I\u0027ve done, I think it could prove tremendously usuful. At the moment, this is all the data I could get in a short period of time.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10540,"name":"Clayton O\u0027Neill","email":"clayton.oneill@charter.com","username":"coneill"},"change_message_id":"960c7a3c198c3ed8452e9537e29710d2e96e267b","unresolved":false,"context_lines":[{"line_number":22,"context_line":"- a change in a common dependency (usually concat)"},{"line_number":23,"context_line":"- idempotency not respected"},{"line_number":24,"context_line":"- backwards incompatibility"},{"line_number":25,"context_line":"- internal parameters position inconsinstency"},{"line_number":26,"context_line":"- out of scope modification (e.g. ssh configuration modified in swift module)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Functional and integration tests should be added to upcoming beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_d1c13582","line":25,"updated":"2015-05-06 13:47:14.000000000","message":"Typo: inconsistency","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":22,"context_line":"- a change in a common dependency (usually concat)"},{"line_number":23,"context_line":"- idempotency not respected"},{"line_number":24,"context_line":"- backwards incompatibility"},{"line_number":25,"context_line":"- internal parameters position inconsinstency"},{"line_number":26,"context_line":"- out of scope modification (e.g. ssh configuration modified in swift module)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Functional and integration tests should be added to upcoming beaker-rspec"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_97c72812","line":25,"in_reply_to":"1aca2d91_d1c13582","updated":"2015-05-06 17:45:42.000000000","message":"Done here and in other three places, thanks.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"a14adda1841c1264481e0a123c0036aa4012cab3","unresolved":false,"context_lines":[{"line_number":37,"context_line":"- A test to detect changes in dependency. Should call a reasonable amount of"},{"line_number":38,"context_line":"  classes that use dependencies as much as possible."},{"line_number":39,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":40,"context_line":"  three times and check for errors"},{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_9d6fc32a","line":40,"updated":"2015-05-05 20:30:21.000000000","message":"++","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"a14adda1841c1264481e0a123c0036aa4012cab3","unresolved":false,"context_lines":[{"line_number":39,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":40,"context_line":"  three times and check for errors"},{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_3da8af76","line":42,"updated":"2015-05-05 20:30:21.000000000","message":"Functional testing already does this - if changing the tests is part of the change then that\u0027s a red flag for backwards incompatibility. I don\u0027t think there needs to be two rounds of testing for this.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"50bd6d96188cbded5067225dde5ecbb2931c6cac","unresolved":false,"context_lines":[{"line_number":39,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":40,"context_line":"  three times and check for errors"},{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_2c0690ce","line":42,"in_reply_to":"1aca2d91_36096d2d","updated":"2015-05-07 19:40:31.000000000","message":"Maybe this bullet point could be rephrased as checking whether the patch affects being able to upgrade from current master to the patch under test, or from latest stable branch to patch under test. I don\u0027t think beaker is especially well suited for that but it would probably be hackable.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"178706146865c2c5d8a3c2f8c68b66814b55fa0b","unresolved":false,"context_lines":[{"line_number":39,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":40,"context_line":"  three times and check for errors"},{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_36096d2d","line":42,"in_reply_to":"1aca2d91_ba2c49ac","updated":"2015-05-06 20:05:56.000000000","message":"If a change is made to a manifest and the tests no longer pass, reporting things along the lines of \"invalid parameter foo\", that means the change was backwards incompatible. That failure should tell us what we need to do, e.g. keep the parameter and add a deprecation warning.\n\nMaybe I misunderstand what you mean by launching a \"first round of test using test from unchanged version\". It would be fine to have two sets of tests, one making sure the deprecated parameters work and give warnings and one making sure the new parameters work. But two stages testing pre- and post-patch doesn\u0027t really make sense, which was how I was reading that.\n\nI don\u0027t think backwards incompatibility is something to be afraid of or something the tests should be built to prevent, it\u0027s just something we need to handle correctly, and should mostly be done via code review.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":39,"context_line":"- A test for idempotency. Should be simple, reapply the same generic manifest"},{"line_number":40,"context_line":"  three times and check for errors"},{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_ba2c49ac","line":42,"in_reply_to":"3acd31a7_3da8af76","updated":"2015-05-06 17:45:42.000000000","message":"What is the meaning, in terms of actions taken, of a red flag ? What happens when a backwards incompatibility is detected ? The change will not be merged ? will be delayed until \u003csomething\u003e happens ? will be merged, but someone will get notified ? Is there a document somewhere that specifies the behaviour ?","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"a14adda1841c1264481e0a123c0036aa4012cab3","unresolved":false,"context_lines":[{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_1da3b3b3","line":44,"updated":"2015-05-05 20:30:21.000000000","message":"I think the rspec-puppet tests cover this sufficiently, e.g. http://git.openstack.org/cgit/stackforge/puppet-keystone/tree/spec/classes/keystone_spec.rb#n159\n\nIf this isn\u0027t sufficient I would rather try to address it in the unit tests and not duplicate tests in rspec-puppet and beaker-rspec.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":5241,"name":"Martin Magr","email":"mmagr@redhat.com","username":"mmagr"},"change_message_id":"3ea292fd3c92672633db0d954cc09f7c29a6d8e8","unresolved":false,"context_lines":[{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9af37de9_4a18e984","line":44,"in_reply_to":"1aca2d91_367ccd71","updated":"2015-05-11 10:50:04.000000000","message":"I must agree with Colleen here. Testing resulting config files brings no value in upstream level.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"178706146865c2c5d8a3c2f8c68b66814b55fa0b","unresolved":false,"context_lines":[{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_367ccd71","line":44,"in_reply_to":"1aca2d91_465d60e8","updated":"2015-05-06 20:05:56.000000000","message":"The phrase \"parameters are created exactly where expected\" confused me. This issue in that patch wasn\u0027t that the parameter moved around in the file, it was that it changed sections and the glance_image provider no longer worked. I think this would have been caught with a basic test for the glance_image provider like https://review.openstack.org/#/c/176521/19/spec/acceptance/basic_glance_spec.rb . I still think the rspec-puppet tests for the ini configs are sufficient and should not be duplicated in beaker-rspec.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":41,"context_line":"- To detect backwards incompatibility it may be useful to launch a first round"},{"line_number":42,"context_line":"  of test using test from unchanged version"},{"line_number":43,"context_line":"- To detect parameters inconsinstency in ini files, it could be useful to check"},{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_465d60e8","line":44,"in_reply_to":"3acd31a7_1da3b3b3","updated":"2015-05-06 17:45:42.000000000","message":"The survey provided me with an example of this kind of problems: https://review.openstack.org/#/c/174455/\nIf the solution to this should not be addressed in beaker-rspec, it could be better to expand this blueprint to cover all groups of tests instead, define the scope for each group, and assign recurrent problems to a group. Then this could even become the place to put red flags behaviour","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"a14adda1841c1264481e0a123c0036aa4012cab3","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Alternatives"},{"line_number":50,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_9d7ea317","line":47,"updated":"2015-05-05 20:30:21.000000000","message":"I think adding a new module dependency should be a red flag, but this is more of a code review issue than an automation issue.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"change_message_id":"178706146865c2c5d8a3c2f8c68b66814b55fa0b","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Alternatives"},{"line_number":50,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_9673a14d","line":47,"in_reply_to":"1aca2d91_a0badcdf","updated":"2015-05-06 20:05:56.000000000","message":"I have no idea how that could be detected in a test.","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  if configuration parameters are created exactly where expected."},{"line_number":45,"context_line":"- Detecting out of scope modification may be challenging. A test should define"},{"line_number":46,"context_line":"  a scope, then track all the modifications done, and notifies all unrecognized"},{"line_number":47,"context_line":"  ones"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Alternatives"},{"line_number":50,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_a0badcdf","line":47,"in_reply_to":"3acd31a7_9d7ea317","updated":"2015-05-06 17:45:42.000000000","message":"Maybe sometimes is not an entire dependency, it\u0027s just a line change in a configuration that shouldn\u0027t be modified by that module","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"3805f9e43bc3a8f6038659af905e24fe8024677e","unresolved":false,"context_lines":[{"line_number":79,"context_line":"Developer impact"},{"line_number":80,"context_line":"----------------"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Developers will need to add tests to fullfill the testing requirements and"},{"line_number":83,"context_line":"cover most of the scenarios"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_bc596537","line":82,"updated":"2015-05-05 18:26:24.000000000","message":"fulfill","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":10022,"name":"Gabriele Cerami","email":"gcerami@redhat.com","username":"panda"},"change_message_id":"8015c2cb2051f24f3378bf5dcdcd87bf04190c65","unresolved":false,"context_lines":[{"line_number":79,"context_line":"Developer impact"},{"line_number":80,"context_line":"----------------"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Developers will need to add tests to fullfill the testing requirements and"},{"line_number":83,"context_line":"cover most of the scenarios"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_5712b063","line":82,"in_reply_to":"3acd31a7_bc596537","updated":"2015-05-06 17:45:42.000000000","message":"Done","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":3153,"name":"Emilien Macchi","email":"emilien@redhat.com","username":"emilienm"},"change_message_id":"0ef0f42e85a27da6d106847fa55da206ff5b239b","unresolved":false,"context_lines":[{"line_number":89,"context_line":"-----------"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"Primary assignee:"},{"line_number":92,"context_line":"  EmilienM"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Other contributors:"},{"line_number":95,"context_line":"  nibalizer"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1aca2d91_790448bf","line":92,"updated":"2015-05-07 19:16:50.000000000","message":"@gabiele, you may want to put your name :-)","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"3805f9e43bc3a8f6038659af905e24fe8024677e","unresolved":false,"context_lines":[{"line_number":102,"context_line":"* finish basic structures \u0026 basic tests:"},{"line_number":103,"context_line":"   - clone modules and test dependencies"},{"line_number":104,"context_line":"   - compile catalog with all classes"},{"line_number":105,"context_line":"   - fix orchestrations issues"},{"line_number":106,"context_line":"   - ensure idempotency"},{"line_number":107,"context_line":"   - basic serverspec, (e.g. nestat/API)"},{"line_number":108,"context_line":"* backwards compatibility"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_bc7045a5","line":105,"updated":"2015-05-05 18:26:24.000000000","message":"what do you mean by this?","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"3805f9e43bc3a8f6038659af905e24fe8024677e","unresolved":false,"context_lines":[{"line_number":111,"context_line":"   - stabilization, documentation, adoption"},{"line_number":112,"context_line":"* parameter inconsinstency"},{"line_number":113,"context_line":"* out of scope modifications"},{"line_number":114,"context_line":"* Integration with tempest"},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Dependencies"},{"line_number":117,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_9cb1094a","line":114,"updated":"2015-05-05 18:26:24.000000000","message":"why?","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"},{"author":{"_account_id":3153,"name":"Emilien Macchi","email":"emilien@redhat.com","username":"emilienm"},"change_message_id":"0d1a8e20e2a6995e1f52c79ef499c076b21a68ad","unresolved":false,"context_lines":[{"line_number":111,"context_line":"   - stabilization, documentation, adoption"},{"line_number":112,"context_line":"* parameter inconsinstency"},{"line_number":113,"context_line":"* out of scope modifications"},{"line_number":114,"context_line":"* Integration with tempest"},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Dependencies"},{"line_number":117,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3acd31a7_6879c4c7","line":114,"in_reply_to":"3acd31a7_9cb1094a","updated":"2015-05-05 23:03:40.000000000","message":"Matt, see discussion here: http://osdir.com/ml/openstack-dev/2015-04/msg01981.html\n\nGabriele, you may want add this discussion in URLs?","commit_id":"82e86ba2eb3a6a70dab6cf134854b6f491267b23"}]}