)]}'
{"tempest/README.rst":[{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"c1303319383460f2ac1632516d3453af310d1163","unresolved":false,"context_lines":[{"line_number":16,"context_line":"   api/ - API tests"},{"line_number":17,"context_line":"   cli/ - CLI tests"},{"line_number":18,"context_line":"   scenario/ - complex scenario tests"},{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Each of these directories have specific rules for what\u0027s allowed content."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff4A%3D","line":19,"updated":"2013-05-07 19:37:16.000000000","message":"We should include performance/ though we don\u0027t have any such tests yet.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"6d16ae2c7a609bea68bdebfa4a117e40d3c21412","unresolved":false,"context_lines":[{"line_number":16,"context_line":"   api/ - API tests"},{"line_number":17,"context_line":"   cli/ - CLI tests"},{"line_number":18,"context_line":"   scenario/ - complex scenario tests"},{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Each of these directories have specific rules for what\u0027s allowed content."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2FfzM%3D","line":19,"in_reply_to":"AAAAOn%2F%2Ff4A%3D","updated":"2013-05-07 20:26:38.000000000","message":"I don\u0027t want to include things we don\u0027t yet have. We can add a directory later, but I\u0027d rather not have empty shells that we don\u0027t already have tests for.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"e621c7d77b39f53da64ab222edde7c241593e8a9","unresolved":false,"context_lines":[{"line_number":18,"context_line":"   scenario/ - complex scenario tests"},{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Each of these directories have specific rules for what\u0027s allowed content."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"3rdparty"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff0M%3D","line":21,"updated":"2013-05-07 20:12:02.000000000","message":"Suggestion for rewording:\n\n Each of these directories contain a different type of test, and instructions\n for the rules and example test cases for that directory are contained in a\n README.rst file in the directory\u0027s root.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"6d16ae2c7a609bea68bdebfa4a117e40d3c21412","unresolved":false,"context_lines":[{"line_number":18,"context_line":"   scenario/ - complex scenario tests"},{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Each of these directories have specific rules for what\u0027s allowed content."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"3rdparty"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2FfzI%3D","line":21,"in_reply_to":"AAAAOn%2F%2Ff0M%3D","updated":"2013-05-07 20:26:38.000000000","message":"good idea.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"c1303319383460f2ac1632516d3453af310d1163","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"cli"},{"line_number":48,"context_line":"------------"},{"line_number":49,"context_line":"CLI tests use the openstack CLI to interact with the OpenStack"},{"line_number":50,"context_line":"cloud. CLI testing in unit tests is somewhat difficult because unlike"},{"line_number":51,"context_line":"server testing, there is no access to server code to"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff4M%3D","line":48,"updated":"2013-05-07 19:37:16.000000000","message":"We should really include the python API in these tests. Perhaps this dir should be called \"clients\" and have \"cli\" and \"python\" subdirs.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"6d16ae2c7a609bea68bdebfa4a117e40d3c21412","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"cli"},{"line_number":48,"context_line":"------------"},{"line_number":49,"context_line":"CLI tests use the openstack CLI to interact with the OpenStack"},{"line_number":50,"context_line":"cloud. CLI testing in unit tests is somewhat difficult because unlike"},{"line_number":51,"context_line":"server testing, there is no access to server code to"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ffy8%3D","line":48,"in_reply_to":"AAAAOn%2F%2Ff0I%3D","updated":"2013-05-07 20:26:38.000000000","message":"so, we don\u0027t have those tests yet, and there was this interesting idea of figuring out a way to make our API tests multiplex into python cli in addition to our clients. So unless there is a strong point of view on adding those here, I\u0027d rather start with what we have.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"e621c7d77b39f53da64ab222edde7c241593e8a9","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"cli"},{"line_number":48,"context_line":"------------"},{"line_number":49,"context_line":"CLI tests use the openstack CLI to interact with the OpenStack"},{"line_number":50,"context_line":"cloud. CLI testing in unit tests is somewhat difficult because unlike"},{"line_number":51,"context_line":"server testing, there is no access to server code to"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff0I%3D","line":48,"in_reply_to":"AAAAOn%2F%2Ff4M%3D","updated":"2013-05-07 20:12:02.000000000","message":"Agree with David, that is a good suggestion.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":1849,"name":"Joe Gordon","email":"joe.gordon0@gmail.com","username":"jogo"},"change_message_id":"acd05cca056d1ecc056e61c47f0f94581506e0c4","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"cli"},{"line_number":48,"context_line":"------------"},{"line_number":49,"context_line":"CLI tests use the openstack CLI to interact with the OpenStack"},{"line_number":50,"context_line":"cloud. CLI testing in unit tests is somewhat difficult because unlike"},{"line_number":51,"context_line":"server testing, there is no access to server code to"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ffo0%3D","line":48,"in_reply_to":"AAAAOn%2F%2Ffy8%3D","updated":"2013-05-07 21:59:27.000000000","message":"I agree with Sean, I would prefer to see the API test use the python CLI as an alternate backend.\n\nAlso, I want to keep the scope of the CLI tests fairly constrained. At least until they are more extensive.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"c1303319383460f2ac1632516d3453af310d1163","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"scenario"},{"line_number":59,"context_line":"------------"},{"line_number":60,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":61,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff4I%3D","line":59,"updated":"2013-05-07 19:37:16.000000000","message":"We have talked about \"integration\" tests that make sure component changes don\u0027t break other components. I presume that is included here and if so we should use that word somewhere in here.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"6d16ae2c7a609bea68bdebfa4a117e40d3c21412","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"scenario"},{"line_number":59,"context_line":"------------"},{"line_number":60,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":61,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ffy4%3D","line":59,"in_reply_to":"AAAAOn%2F%2Ff0A%3D","updated":"2013-05-07 20:26:38.000000000","message":"+1 to Jay","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"e621c7d77b39f53da64ab222edde7c241593e8a9","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"scenario"},{"line_number":59,"context_line":"------------"},{"line_number":60,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":61,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff0A%3D","line":59,"in_reply_to":"AAAAOn%2F%2Ff4I%3D","updated":"2013-05-07 20:12:02.000000000","message":"I think we wanted to get away from the term \"integration\" since people have different ideas about what an integration test is... I like Sean\u0027s description of what a scenario test is below.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"de0b5d0c3b4c42cd6cd47d27723f04ea37713fc5","unresolved":false,"context_lines":[{"line_number":61,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"TODO: tests/network/test_network_basic_ops.py,"},{"line_number":67,"context_line":"tests/compute/servers/*_ops.py should move to tempest/scenario (others)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2FfTc%3D","line":64,"updated":"2013-05-08 02:56:39.000000000","message":"I\u0027d like to know the meaning of \u0027the OpenStack python clients\u0027. It means \u0027/usr/local/bin/openstack\u0027 or \u0027/usr/local/bin/{nova,keystone,etc.}\u0027 or something else?\nIf it means \u0027/usr/local/bin/openstack\u0027, the command does not support glance now, and I think it is difficult to write scenario tests.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":5803,"name":"Attila Fazekas","email":"afazekas@redhat.com","username":"afazekas"},"change_message_id":"1895996c144c08c438160d17f936f78c59d2ecfe","unresolved":false,"context_lines":[{"line_number":61,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"TODO: tests/network/test_network_basic_ops.py,"},{"line_number":67,"context_line":"tests/compute/servers/*_ops.py should move to tempest/scenario (others)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Fe9Q%3D","line":64,"in_reply_to":"AAAAOn%2F%2FfTc%3D","updated":"2013-05-08 08:38:05.000000000","message":"It basically means you are using the client libraries  like: import keystoneclient.\n\nAs I remember on the summit we saidwe should use the CLI tools for complex scenario tests. It should be suggested here as well.\n\nIMHO we should be less picky in tools for complex test cases.\n\nWe should use the clients/cli folder for basic cli tests just to make sure, nobody broken the cli clients very badly.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":5803,"name":"Attila Fazekas","email":"afazekas@redhat.com","username":"afazekas"},"change_message_id":"1895996c144c08c438160d17f936f78c59d2ecfe","unresolved":false,"context_lines":[{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"TODO: tests/network/test_network_basic_ops.py,"},{"line_number":67,"context_line":"tests/compute/servers/*_ops.py should move to tempest/scenario (others)"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2FfAY%3D","line":65,"updated":"2013-05-08 08:38:05.000000000","message":"We should have several sub-directories in the scenario directory for a specific environment, for example tests for a specific multi node deployment.\n\nThese folders must contain a detailed description about the environment where they expected to pass.\n\nSome of these test cases may contain remote host machine reboot steps.\n\nWe should use the \"gate\" attibute for gating scenario tests.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"2d76e81e1e32e8af21833a4b694174ddb9eba9fc","unresolved":false,"context_lines":[{"line_number":62,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"TODO: tests/network/test_network_basic_ops.py,"},{"line_number":67,"context_line":"tests/compute/servers/*_ops.py should move to tempest/scenario (others)"},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2FewY%3D","line":65,"in_reply_to":"AAAAOn%2F%2FfAY%3D","updated":"2013-05-08 11:25:51.000000000","message":"I think we can let scenarios grow over time, I don\u0027t want to predefine those right now. I agree that we\u0027re going to need a gate attribute once we open this up.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"c1303319383460f2ac1632516d3453af310d1163","unresolved":false,"context_lines":[{"line_number":74,"context_line":"be provided to help detect breaks (stack traces in the logs)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Open Questions: where does white box testing go?"},{"line_number":78,"context_line":" * proposal #1 - white box is an optional extension on top of scenario"},{"line_number":79,"context_line":"   testing"},{"line_number":80,"context_line":" * proposal #2 - white box tests are their own directory"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ff4E%3D","line":77,"updated":"2013-05-07 19:37:16.000000000","message":"I would put them in their own directory since they really have nothing to do with scenario vs api. They could augment either kind of test I think.","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"6d16ae2c7a609bea68bdebfa4a117e40d3c21412","unresolved":false,"context_lines":[{"line_number":74,"context_line":"be provided to help detect breaks (stack traces in the logs)."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Open Questions: where does white box testing go?"},{"line_number":78,"context_line":" * proposal #1 - white box is an optional extension on top of scenario"},{"line_number":79,"context_line":"   testing"},{"line_number":80,"context_line":" * proposal #2 - white box tests are their own directory"}],"source_content_type":"text/x-rst","patch_set":2,"id":"AAAAOn%2F%2Ffy0%3D","line":77,"in_reply_to":"AAAAOn%2F%2Ff4E%3D","updated":"2013-05-07 20:26:38.000000000","message":"I\u0027m good with separate directory","commit_id":"7a3f9974089b0d748c8c05f1a3a4ea3dbac41059"},{"author":{"_account_id":1795,"name":"Ravikumar Venkatesan","email":"ravikumar.venkatesan@hpe.com","username":"ravikumar-venkatesan"},"change_message_id":"8f171e7a80e866fb46633ec173b643b70d6c7479","unresolved":false,"context_lines":[{"line_number":18,"context_line":"   scenario/ - complex scenario tests"},{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":"   whitebox/ - white box testing"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Each of these directories contain a different type of test, and instructions"},{"line_number":23,"context_line":"for the rules and example test cases for that directory are contained in a"},{"line_number":24,"context_line":"README.rst file in the directory\u0027s root."}],"source_content_type":"text/x-rst","patch_set":3,"id":"AAAAOn%2F%2FdtA%3D","line":21,"updated":"2013-05-08 21:07:03.000000000","message":"I will create one more folder\nperformance \nJust a placeholder for now","commit_id":"2da3afe661238426110ed3861b08d6109aada11e"},{"author":{"_account_id":1795,"name":"Ravikumar Venkatesan","email":"ravikumar.venkatesan@hpe.com","username":"ravikumar-venkatesan"},"change_message_id":"8f171e7a80e866fb46633ec173b643b70d6c7479","unresolved":false,"context_lines":[{"line_number":28,"context_line":"------------"},{"line_number":29,"context_line":"Many openstack components include 3rdparty API support. It is"},{"line_number":30,"context_line":"completely legitmate for Tempest to include tests of 3rdparty APIs,"},{"line_number":31,"context_line":"but those should be kept seperate from normal OpenStack validation."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"TODO: tempest/tests/boto should become tempest/3rdparty/boto"},{"line_number":34,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"AAAAOn%2F%2FduY%3D","line":31,"updated":"2013-05-08 21:07:03.000000000","message":"Those tests are not part of gated tests and not part of nightly regress, and can be run on adhoc basis.","commit_id":"2da3afe661238426110ed3861b08d6109aada11e"},{"author":{"_account_id":1795,"name":"Ravikumar Venkatesan","email":"ravikumar.venkatesan@hpe.com","username":"ravikumar-venkatesan"},"change_message_id":"8f171e7a80e866fb46633ec173b643b70d6c7479","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"api"},{"line_number":37,"context_line":"------------"},{"line_number":38,"context_line":"API tests are validation tests for the OpenStack API. They should not"},{"line_number":39,"context_line":"used the existing python clients for OpenStack, but should instead use"},{"line_number":40,"context_line":"the tempest implementations of clients. This allows us to test both"},{"line_number":41,"context_line":"XML, and invalid JSON effectively."}],"source_content_type":"text/x-rst","patch_set":3,"id":"AAAAOn%2F%2FduE%3D","line":38,"updated":"2013-05-08 21:07:03.000000000","message":"API tests are functiona/integration validation tests for the OpenStack API. Those tests are part of the gate.","commit_id":"2da3afe661238426110ed3861b08d6109aada11e"},{"author":{"_account_id":1795,"name":"Ravikumar Venkatesan","email":"ravikumar.venkatesan@hpe.com","username":"ravikumar-venkatesan"},"change_message_id":"8f171e7a80e866fb46633ec173b643b70d6c7479","unresolved":false,"context_lines":[{"line_number":38,"context_line":"API tests are validation tests for the OpenStack API. They should not"},{"line_number":39,"context_line":"used the existing python clients for OpenStack, but should instead use"},{"line_number":40,"context_line":"the tempest implementations of clients. This allows us to test both"},{"line_number":41,"context_line":"XML, and invalid JSON effectively."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"When it makes sense, API testing should be moved closer to the"},{"line_number":44,"context_line":"projects themselves, possibly as functional tests in their unit test"}],"source_content_type":"text/x-rst","patch_set":3,"id":"AAAAOn%2F%2Fdto%3D","line":41,"updated":"2013-05-08 21:07:03.000000000","message":"This allows us to test both XML, and JSON format effectively.","commit_id":"2da3afe661238426110ed3861b08d6109aada11e"},{"author":{"_account_id":1795,"name":"Ravikumar Venkatesan","email":"ravikumar.venkatesan@hpe.com","username":"ravikumar-venkatesan"},"change_message_id":"8f171e7a80e866fb46633ec173b643b70d6c7479","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":64,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":65,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"TODO: tests/network/test_network_basic_ops.py,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"AAAAOn%2F%2FdtU%3D","line":66,"updated":"2013-05-08 21:07:03.000000000","message":"The scenario tests are not to be part of gated tests, and can be run on needed basis.\nThe implementation of Scenario tests is left to the test developer/user.","commit_id":"2da3afe661238426110ed3861b08d6109aada11e"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"e0cbd907efaacecc2f2aeb27185d4b089d590f66","unresolved":false,"context_lines":[{"line_number":19,"context_line":"   stress/ - stress tests"},{"line_number":20,"context_line":"   whitebox/ - white box testing"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Each of these directories contain a different type of test, and instructions"},{"line_number":23,"context_line":"for the rules and example test cases for that directory are contained in a"},{"line_number":24,"context_line":"README.rst file in the directory\u0027s root."},{"line_number":25,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaJ8%3D","line":22,"updated":"2013-05-10 15:34:17.000000000","message":"contains? tests? \", and\"?","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"e0cbd907efaacecc2f2aeb27185d4b089d590f66","unresolved":false,"context_lines":[{"line_number":20,"context_line":"   whitebox/ - white box testing"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Each of these directories contain a different type of test, and instructions"},{"line_number":23,"context_line":"for the rules and example test cases for that directory are contained in a"},{"line_number":24,"context_line":"README.rst file in the directory\u0027s root."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaJ0%3D","line":23,"updated":"2013-05-10 15:34:17.000000000","message":"instructions for the rules? or instructions, rules and ...","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"e0cbd907efaacecc2f2aeb27185d4b089d590f66","unresolved":false,"context_lines":[{"line_number":28,"context_line":"------------"},{"line_number":29,"context_line":"Many openstack components include 3rdparty API support. It is"},{"line_number":30,"context_line":"completely legitmate for Tempest to include tests of 3rdparty APIs,"},{"line_number":31,"context_line":"but those should be kept seperate from normal OpenStack validation."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"TODO: tempest/tests/boto should become tempest/3rdparty/boto"},{"line_number":34,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaJw%3D","line":31,"updated":"2013-05-10 15:34:17.000000000","message":"from *the* normal?","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"e0cbd907efaacecc2f2aeb27185d4b089d590f66","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"api"},{"line_number":37,"context_line":"------------"},{"line_number":38,"context_line":"API tests are validation tests for the OpenStack API. They should not"},{"line_number":39,"context_line":"used the existing python clients for OpenStack, but should instead use"},{"line_number":40,"context_line":"the tempest implementations of clients. This allows us to test both"},{"line_number":41,"context_line":"XML, and invalid JSON effectively."}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaJg%3D","line":38,"updated":"2013-05-10 15:34:17.000000000","message":"should not *use* the clients?","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"e0cbd907efaacecc2f2aeb27185d4b089d590f66","unresolved":false,"context_lines":[{"line_number":38,"context_line":"API tests are validation tests for the OpenStack API. They should not"},{"line_number":39,"context_line":"used the existing python clients for OpenStack, but should instead use"},{"line_number":40,"context_line":"the tempest implementations of clients. This allows us to test both"},{"line_number":41,"context_line":"XML, and invalid JSON effectively."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"When it makes sense, API testing should be moved closer to the"},{"line_number":44,"context_line":"projects themselves, possibly as functional tests in their unit test"}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaJY%3D","line":41,"updated":"2013-05-10 15:34:17.000000000","message":"test both XML, and *invalid*?? JSON","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"876180a417ee9d572d0017b5c7589ff45c99afb1","unresolved":false,"context_lines":[{"line_number":38,"context_line":"API tests are validation tests for the OpenStack API. They should not"},{"line_number":39,"context_line":"used the existing python clients for OpenStack, but should instead use"},{"line_number":40,"context_line":"the tempest implementations of clients. This allows us to test both"},{"line_number":41,"context_line":"XML, and invalid JSON effectively."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"When it makes sense, API testing should be moved closer to the"},{"line_number":44,"context_line":"projects themselves, possibly as functional tests in their unit test"}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaGI%3D","line":41,"in_reply_to":"AAAAOn%2F%2FaJY%3D","updated":"2013-05-10 15:51:57.000000000","message":"yes, invalid JSON. We can push invalid JSON via the raw clients, not via the native clients.","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"eddd69fdb21f18603a0c5ecbe6c8bd2bae42e433","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"scenario"},{"line_number":62,"context_line":"------------"},{"line_number":63,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":64,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":65,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaIk%3D","line":63,"updated":"2013-05-10 15:50:15.000000000","message":"\"through path\" or maybe thorough test?","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"eddd69fdb21f18603a0c5ecbe6c8bd2bae42e433","unresolved":false,"context_lines":[{"line_number":62,"context_line":"------------"},{"line_number":63,"context_line":"Scenario tests are complex through path tests for OpenStack"},{"line_number":64,"context_line":"functionality. They are typically a series of steps where complicated"},{"line_number":65,"context_line":"state requiring multiple services is set up exercised, and torn down."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Scenario tests can and should use the OpenStack python clients."},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaG0%3D","line":65,"updated":"2013-05-10 15:50:15.000000000","message":"setup of complicated state, which requires multiple services, is excercised and torn down.\n\nNot sure here, this still seems weird to me. What is exercised - setup, state, ... \n\nAt least \u0027services are\u0027 or \u0027state/setup is\u0027.","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"},{"author":{"_account_id":6683,"name":"Pavel Sedlák","email":"psedlak@redhat.com","username":"psedlak"},"change_message_id":"eddd69fdb21f18603a0c5ecbe6c8bd2bae42e433","unresolved":false,"context_lines":[{"line_number":74,"context_line":"-----------"},{"line_number":75,"context_line":"Stress tests are designed to stress an OpenStack environment by"},{"line_number":76,"context_line":"running a high workload against it and seeing what breaks. Tools may"},{"line_number":77,"context_line":"be provided to help detect breaks (stack traces in the logs)."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"TODO: old stress tests deleted, new_stress that david is working on"},{"line_number":80,"context_line":"moves into here."}],"source_content_type":"text/x-rst","patch_set":5,"id":"AAAAOn%2F%2FaGU%3D","line":77,"updated":"2013-05-10 15:50:15.000000000","message":"definitely not sure here too, just \u0027breaks\u0027 and \u0027seeing\u0027 here sounds weird, but ...","commit_id":"62bb483a6c8f9e719fbdd6163425b9138f5d255b"}]}