)]}'
{"specs/tempest/ddt-testing.rst":[{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"026646f63beb67b9f836f291ad0b22f7dfc1e396","unresolved":false,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Currently we see tests with for-loop that iterate through different"},{"line_number":19,"context_line":"data set. This can be problematic for debugging and in general"},{"line_number":20,"context_line":"for test separation."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_5113f45c","line":20,"range":{"start_line":20,"start_character":9,"end_line":20,"end_character":19},"updated":"2015-12-22 15:21:58.000000000","message":"Do you have an example of these test cases. I\u0027m just curious, I think I know the ones you\u0027re talking about, but just pointing out would be good.","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":7872,"name":"Marc Koderer","email":"marc@koderer.com","username":"mkoderer"},"change_message_id":"88ff006de8716b21cc744ceee6f5666d6107d485","unresolved":false,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Currently we see tests with for-loop that iterate through different"},{"line_number":19,"context_line":"data set. This can be problematic for debugging and in general"},{"line_number":20,"context_line":"for test separation."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_7310e659","line":20,"range":{"start_line":20,"start_character":9,"end_line":20,"end_character":19},"in_reply_to":"1a7b0d38_5113f45c","updated":"2015-12-28 10:47:28.000000000","message":"Done","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"026646f63beb67b9f836f291ad0b22f7dfc1e396","unresolved":false,"context_lines":[{"line_number":42,"context_line":"look like::"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    @ddt.data((3, 2), (3, 0), (8, 0), (0, 0), (11, None))"},{"line_number":45,"context_line":"    def test_create_security_group_rule_with_icmp_type_code(self, type_codes):"},{"line_number":46,"context_line":"        self._create_verify_security_group_rule(sg_id, direction,"},{"line_number":47,"context_line":"                                                self.ethertype, protocol,"},{"line_number":48,"context_line":"                                                type_codes[0], type_codes[1])"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_91407c09","line":45,"range":{"start_line":45,"start_character":8,"end_line":45,"end_character":59},"updated":"2015-12-22 15:21:58.000000000","message":"So my only question is how does this interact with setUp() and tearDown(). I assume because it\u0027s dynamically generating new test methods we will re-run both setUp() and tearDown() (and addCleanup()) more frequently now. Do you think that will have a performance penalty?","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":7872,"name":"Marc Koderer","email":"marc@koderer.com","username":"mkoderer"},"change_message_id":"88ff006de8716b21cc744ceee6f5666d6107d485","unresolved":false,"context_lines":[{"line_number":42,"context_line":"look like::"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    @ddt.data((3, 2), (3, 0), (8, 0), (0, 0), (11, None))"},{"line_number":45,"context_line":"    def test_create_security_group_rule_with_icmp_type_code(self, type_codes):"},{"line_number":46,"context_line":"        self._create_verify_security_group_rule(sg_id, direction,"},{"line_number":47,"context_line":"                                                self.ethertype, protocol,"},{"line_number":48,"context_line":"                                                type_codes[0], type_codes[1])"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_21c76897","line":45,"range":{"start_line":45,"start_character":8,"end_line":45,"end_character":59},"in_reply_to":"1a7b0d38_91407c09","updated":"2015-12-28 10:47:28.000000000","message":"I added some words about it in the new patchset. Basically I don\u0027t see any issue with performance since resouce setup is mainly done within class level.","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"47a5299bed685187429d7e33125709baa7d1de9d","unresolved":false,"context_lines":[{"line_number":56,"context_line":"If so it hashes the currently used data set (using pickle) and add\u0027s it to the"},{"line_number":57,"context_line":"given idempotent_id. There for all idempotent_id will be unique again but"},{"line_number":58,"context_line":"longer in case of DDT."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Implementation"},{"line_number":61,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a7e1126_0abd49d7","line":59,"updated":"2015-12-22 07:40:04.000000000","message":"hmm, I\u0027d like to clarify here.\n\nI think tests which generated dynamically should have same idempotent_ids.\n\nI mean for example,\n test_create_security_group_rule_with_icmp_type_code_4__0__0\nand\n test_create_security_group_rule_with_icmp_type_code_5__11__None\nshould have same idempotent_ids, IMO.\nBecause they are same test case but only different data.\n\nAnd, current behavior is same the above.\nSo I think no issue is here.","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":7872,"name":"Marc Koderer","email":"marc@koderer.com","username":"mkoderer"},"change_message_id":"88ff006de8716b21cc744ceee6f5666d6107d485","unresolved":false,"context_lines":[{"line_number":56,"context_line":"If so it hashes the currently used data set (using pickle) and add\u0027s it to the"},{"line_number":57,"context_line":"given idempotent_id. There for all idempotent_id will be unique again but"},{"line_number":58,"context_line":"longer in case of DDT."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Implementation"},{"line_number":61,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_41c4f48f","line":59,"in_reply_to":"1a7b0d38_ae167753","updated":"2015-12-28 10:47:28.000000000","message":"Done","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"026646f63beb67b9f836f291ad0b22f7dfc1e396","unresolved":false,"context_lines":[{"line_number":56,"context_line":"If so it hashes the currently used data set (using pickle) and add\u0027s it to the"},{"line_number":57,"context_line":"given idempotent_id. There for all idempotent_id will be unique again but"},{"line_number":58,"context_line":"longer in case of DDT."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Implementation"},{"line_number":61,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a7b0d38_ae167753","line":59,"in_reply_to":"3a7e1126_0abd49d7","updated":"2015-12-22 15:21:58.000000000","message":"I agree, the idempotent id is designed to be idempotent per method. If the code being executed in a test method is the same the id should remain consistent. We have this behavior with subclasses and test scenarios already.","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"4841548b6345982a5ae6b8402ca5ce0de4497081","unresolved":false,"context_lines":[{"line_number":76,"context_line":"----------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"* Implement idempotent_id functionality"},{"line_number":79,"context_line":"* Port tests"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"Dependencies"},{"line_number":82,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a7e1126_ea6ebdfe","line":79,"updated":"2015-12-22 08:58:54.000000000","message":"I think we should list up the port tests to know the size of this work.","commit_id":"c723e6f4638225da103f945b34518f8da54e6330"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"caf5e8a2158dc2f606e75502734bf782d2fd04a3","unresolved":false,"context_lines":[{"line_number":55,"context_line":"setup is usually done within class methods which won\u0027t effect the performance."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Idempotent_id and refstack"},{"line_number":59,"context_line":"--------------------------"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Since refstack should handle generated tests under the same id, it should"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa69d971_fb474d75","line":58,"updated":"2016-01-07 16:32:23.000000000","message":"I would just drop this section, but it\u0027s no big deal","commit_id":"695437592e7c2b21a9b2dcee745ac3b0fd8688a3"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"caf5e8a2158dc2f606e75502734bf782d2fd04a3","unresolved":false,"context_lines":[{"line_number":59,"context_line":"--------------------------"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Since refstack should handle generated tests under the same id, it should"},{"line_number":62,"context_line":"be fine to use the idempotent decorator to uniquely identify the test group."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Implementation"},{"line_number":65,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa69d971_9b5df14a","line":62,"updated":"2016-01-07 16:32:23.000000000","message":"Its probably worth expanding this point out by saying that by moving the data to the decorator the idempotent id is doing it\u0027s job by correctly indicating what was previously 1 test in the results stream (with a for loop in the method) after the migration is is the group of tests in the result stream accomplishing the same task. (this was actually a design point of idempotent id)","commit_id":"695437592e7c2b21a9b2dcee745ac3b0fd8688a3"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"8d64b994ec67161a334a1684a0cd7c008366ea6b","unresolved":false,"context_lines":[{"line_number":59,"context_line":"--------------------------"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Since refstack should handle generated tests under the same id, it should"},{"line_number":62,"context_line":"be fine to use the idempotent decorator to uniquely identify the test group."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Implementation"},{"line_number":65,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa69d971_86c8b723","line":62,"in_reply_to":"fa69d971_9b5df14a","updated":"2016-01-10 14:23:35.000000000","message":"agree","commit_id":"695437592e7c2b21a9b2dcee745ac3b0fd8688a3"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"caf5e8a2158dc2f606e75502734bf782d2fd04a3","unresolved":false,"context_lines":[{"line_number":93,"context_line":"1. https://review.openstack.org/#/c/223953/"},{"line_number":94,"context_line":"2. https://pypi.python.org/pypi/ddt"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":".. _config_tempest.py: https://github.com/redhat-openstack/tempest/blob/master/tools/config_tempest.py"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa69d971_bb5a354f","line":96,"updated":"2016-01-07 16:32:23.000000000","message":"I\u0027m not sure you need this external reference, it doesn\u0027t look to be used anywhere","commit_id":"695437592e7c2b21a9b2dcee745ac3b0fd8688a3"},{"author":{"_account_id":5689,"name":"Masayuki Igawa","email":"masayuki@igawa.io","username":"igawa"},"change_message_id":"8d64b994ec67161a334a1684a0cd7c008366ea6b","unresolved":false,"context_lines":[{"line_number":93,"context_line":"1. https://review.openstack.org/#/c/223953/"},{"line_number":94,"context_line":"2. https://pypi.python.org/pypi/ddt"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":".. _config_tempest.py: https://github.com/redhat-openstack/tempest/blob/master/tools/config_tempest.py"}],"source_content_type":"text/x-rst","patch_set":3,"id":"fa69d971_06dda7df","line":96,"in_reply_to":"fa69d971_bb5a354f","updated":"2016-01-10 14:23:35.000000000","message":"hmm, agree also here.","commit_id":"695437592e7c2b21a9b2dcee745ac3b0fd8688a3"}]}