)]}'
{"test/probe/test_reconstructor_rebuild.py":[{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"0e9c62ebd7611028923552bed19c407c414187b7","unresolved":false,"context_lines":[{"line_number":244,"context_line":"                if e.errno !\u003d errno.ENOENT:"},{"line_number":245,"context_line":"                    raise"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"    def _assert_all_nodes_have_frag(self, opart, onodes):"},{"line_number":248,"context_line":"        # check all frags are in place"},{"line_number":249,"context_line":"        failures \u003d []"},{"line_number":250,"context_line":"        for node in onodes:"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_64e50c36","line":247,"updated":"2016-10-11 17:31:20.000000000","message":"wondering if it would make sense to all check if the durable state also got reconstructed.","commit_id":"774db3d3d352314ed5d57e16e594c21531811624"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"2be00fd6ff0c995dde48a8a6a27080435c7b33a5","unresolved":false,"context_lines":[{"line_number":244,"context_line":"                if e.errno !\u003d errno.ENOENT:"},{"line_number":245,"context_line":"                    raise"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"    def _assert_all_nodes_have_frag(self, opart, onodes):"},{"line_number":248,"context_line":"        # check all frags are in place"},{"line_number":249,"context_line":"        failures \u003d []"},{"line_number":250,"context_line":"        for node in onodes:"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_3269568c","line":247,"in_reply_to":"fa6399be_64e50c36","updated":"2016-10-13 00:20:12.000000000","message":"while reviewing this patch and patch 355958, I noticed that test_main in both test_reconstructor_rebuild and test_reconstructor_durable are not correct (they follow the same pattern). The test creates a list of lists to remove either durable state or data file and then test reconstruction, but it actually runs the test on each node individually as opposed to the whole list (see Lines 157-165).\n\nThis new test that Alistair added (i.e., test_rebuild_with_multiple_failures) is actually doing the same thing, but correctly. So my suggestion is, let\u0027s remove test_main in this file, we can completely remove test_reconstructor_durable and we just need to add a test in _assert_all_nodes_have_frag to also check for the durable state.","commit_id":"774db3d3d352314ed5d57e16e594c21531811624"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":150,"context_line":"                os.remove(os.path.join(part_dir, \u0027hashes.pkl\u0027))"},{"line_number":151,"context_line":"            except OSError as e:"},{"line_number":152,"context_line":"                if e.errno !\u003d errno.ENOENT:"},{"line_number":153,"context_line":"                    raise"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"    def _format_node(self, node):"},{"line_number":156,"context_line":"        return \u0027%s#%s\u0027 % (node[\u0027device\u0027], node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_5778ca8f","line":153,"updated":"2016-11-04 22:34:42.000000000","message":"so this was my pause point in the debugger to inspect each scenario in the loop and verify things were getting broken they way they promised.","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":172,"context_line":"            self.fail(\u0027\\n\u0027.join([\u0027    Node %r raised %r\u0027 %"},{"line_number":173,"context_line":"                                 (self._format_node(node), exc)"},{"line_number":174,"context_line":"                                 for (node, exc) in failures]))"},{"line_number":175,"context_line":"        return frag_headers, frag_etags"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"    @contextmanager"},{"line_number":178,"context_line":"    def _annotate_failure_with_scenario(self, failed, non_durable):"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_579feab3","line":175,"updated":"2016-11-04 22:34:42.000000000","message":"this is the best debug/assert/wtf method ever - i want it from the command line ;)","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":184,"context_line":"                \u0027 failed with:\\n%s\u0027 %"},{"line_number":185,"context_line":"                ([self._format_node(self.onodes[n]) for n in failed],"},{"line_number":186,"context_line":"                 [self._format_node(self.onodes[n]) for n in non_durable], err)"},{"line_number":187,"context_line":"            )"},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"    def _test_rebuild_scenario(self, failed, non_durable,"},{"line_number":190,"context_line":"                               reconstructor_cycles):"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_9487c0d6","line":187,"updated":"2016-11-04 22:34:42.000000000","message":"good stuff, this is what a failure looks like with the reconstructor change reverted:\n\nthat \n    FAIL: test_rebuild_with_missing_frags_and_non_durable_frags (test.probe.test_reconstructor_rebuild.TestReconstructorRebuild)\n    ----------------------------------------------------------------------\n    Traceback (most recent call last):\n      File \"/home/ubuntu/swift/test/probe/test_reconstructor_rebuild.py\", line 268, in test_rebuild_with_missing_frags_and_non_durable_frags\n        self._test_rebuild_scenario(failed, non_durable, 2)\n      File \"/home/ubuntu/swift/test/probe/test_reconstructor_rebuild.py\", line 222, in _test_rebuild_scenario\n        self.assertEqual(self.frag_headers, frag_headers)\n      File \"/usr/lib/python2.7/contextlib.py\", line 35, in __exit__\n        self.gen.throw(type, value, traceback)\n      File \"/home/ubuntu/swift/test/probe/test_reconstructor_rebuild.py\", line 186, in _annotate_failure_with_scenario\n        [self._format_node(self.onodes[n]) for n in non_durable], err)\n    AssertionError: Scenario with failed nodes: [u\u0027sdb2#0\u0027, u\u0027sdb7#2\u0027], non-durable nodes: [u\u0027sdb5#1\u0027]\n     failed with:\n        Node u\u0027sdb2#0\u0027 raised DirectClientException(u\"Object server 127.0.0.1:6020 direct GET \u0027/sdb2/631/AUTH_test/container-2b164044-1f4a-4216-ba6e-2f73175a9e02/object-6c25f24a-3b1a-4885-86e3-3c0b44e3fe76\u0027 gave status 404\",)\n        Node u\u0027sdb5#1\u0027 raised DirectClientException(u\"Object server 127.0.0.1:6010 direct GET \u0027/sdb5/631/AUTH_test/container-2b164044-1f4a-4216-ba6e-2f73175a9e02/object-6c25f24a-3b1a-4885-86e3-3c0b44e3fe76\u0027 gave status 404\",)\n        Node u\u0027sdb7#2\u0027 raised DirectClientException(u\"Object server 127.0.0.1:6030 direct GET \u0027/sdb7/631/AUTH_test/container-2b164044-1f4a-4216-ba6e-2f73175a9e02/object-6c25f24a-3b1a-4885-86e3-3c0b44e3fe76\u0027 gave status 404\",)","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":204,"context_line":""},{"line_number":205,"context_line":"        # fire up reconstructor"},{"line_number":206,"context_line":"        for i in range(reconstructor_cycles):"},{"line_number":207,"context_line":"            self.reconstructor.once()"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"        # check GET via proxy returns expected data and metadata"},{"line_number":210,"context_line":"        with self._annotate_failure_with_scenario(failed, non_durable):"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_77f22e08","line":207,"updated":"2016-11-04 22:34:42.000000000","message":"this was my other debug break point (inside the loop) to make sure things were getting fixed up slowly with each run","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":241,"context_line":"        far_nodes \u003d (0, 4)"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"        for non_durable_nodes in [single_node, adj_nodes, far_nodes]:"},{"line_number":244,"context_line":"            self._test_rebuild_scenario([], non_durable_nodes, 1)"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    def test_rebuild_with_missing_frags_and_non_durable_frags(self):"},{"line_number":247,"context_line":"        # pick some nodes with parts deleted, some with non-durable fragments"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_147490fb","line":244,"updated":"2016-11-04 22:34:42.000000000","message":"good call dropping the other test in favor of the format here","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":251,"context_line":"            ((0, 4), (2,)),"},{"line_number":252,"context_line":"        ]"},{"line_number":253,"context_line":"        for failed, non_durable in scenarios:"},{"line_number":254,"context_line":"            self._test_rebuild_scenario(failed, non_durable, 3)"},{"line_number":255,"context_line":"        scenarios \u003d ["},{"line_number":256,"context_line":"            # failed, non-durable"},{"line_number":257,"context_line":"            ((0, 1), (2,)),"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_972a622b","line":254,"updated":"2016-11-04 22:34:42.000000000","message":"nit: because of the way these failures are spaced - doesn\u0027t need to be 3, both of these scenarios rebuild the missing frags on the first sweep","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":255,"context_line":"        scenarios \u003d ["},{"line_number":256,"context_line":"            # failed, non-durable"},{"line_number":257,"context_line":"            ((0, 1), (2,)),"},{"line_number":258,"context_line":"            ((0, 2), (1,)),"},{"line_number":259,"context_line":"        ]"},{"line_number":260,"context_line":"        for failed, non_durable in scenarios:"},{"line_number":261,"context_line":"            # why 2 repeats? consider missing fragment on nodes 0, 1  and"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_f7f0fecb","line":258,"updated":"2016-11-04 22:34:42.000000000","message":"yeah, this is the guy that\u0027s super broken with HEAD^ because no matter how many time\u0027s you loop you don\u0027t ever make node index #1 durable because he\u0027s trapped by an unrebuildable-without-non-durable frags on either side\n\nNow that bug is fixed it might not be that interesting to call this scenario out specifically; but the *idea* of a node (or set of failed nodes) \"trapped\" with failures on either side is important when reasoning about the partner handling of EC reconstruction","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":268,"context_line":"            self._test_rebuild_scenario(failed, non_durable, 2)"},{"line_number":269,"context_line":"        scenarios \u003d ["},{"line_number":270,"context_line":"            # failed, non-durable"},{"line_number":271,"context_line":"            ((0, 2), (1, 3, 5)),"},{"line_number":272,"context_line":"            ((0,), (1, 2, 4, 5)),"},{"line_number":273,"context_line":"        ]"},{"line_number":274,"context_line":"        for failed, non_durable in scenarios:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_17ef32b1","line":271,"updated":"2016-11-04 22:34:42.000000000","message":"so 4 is the only one that\u0027s durable\n\nrun #1 - 3 \u0026 5 made durable by their partner, (surprise for me 2 was rebuilt, probably by 3 after he was made durable)\n\nrun #2 - I\u0027m good now 4, 5 -\u003e 0, 1 \u003c- 2, 3","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":269,"context_line":"        scenarios \u003d ["},{"line_number":270,"context_line":"            # failed, non-durable"},{"line_number":271,"context_line":"            ((0, 2), (1, 3, 5)),"},{"line_number":272,"context_line":"            ((0,), (1, 2, 4, 5)),"},{"line_number":273,"context_line":"        ]"},{"line_number":274,"context_line":"        for failed, non_durable in scenarios:"},{"line_number":275,"context_line":"            # why 3 repeats? consider missing fragment on node 0 and single"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_977182e3","line":272,"updated":"2016-11-04 22:34:42.000000000","message":"so 3 is the hero here\n\nrun #1 - X, 1, 2 \u003c- 3 -\u003e 4, 5\n\nrun #2 - X, 1 \u003c- 2, 3, 4 -\u003e 5\n\nrun #3 - 0 \u003c- 1, 2, 3, 4, 5\n\nawesome","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"}],"test/unit/obj/test_reconstructor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"da595c2ccc9f679bcd8838c7a771bbb1d0913eaa","unresolved":false,"context_lines":[{"line_number":2557,"context_line":"            self.assertIn(\u0027X-Backend-Storage-Policy-Index\u0027, called_header)"},{"line_number":2558,"context_line":"            self.assertEqual(called_header[\u0027X-Backend-Storage-Policy-Index\u0027],"},{"line_number":2559,"context_line":"                             self.policy)"},{"line_number":2560,"context_line":"            self.assertIn(\u0027X-Backend-Fragment-Preferences\u0027, called_header)"},{"line_number":2561,"context_line":"            self.assertEqual("},{"line_number":2562,"context_line":"                [{\u0027timestamp\u0027: \u00271234567890.12345\u0027, \u0027exclude\u0027: []}],"},{"line_number":2563,"context_line":"                json.loads(called_header[\u0027X-Backend-Fragment-Preferences\u0027]))"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba5da102_b4d044f9","line":2560,"updated":"2016-11-04 22:34:42.000000000","message":"not bad\n\n    FAIL: test_reconstruct_fa_no_errors (test.unit.obj.test_reconstructor.TestObjectReconstructor)\n    ----------------------------------------------------------------------\n    Traceback (most recent call last):\n      File \"/home/ubuntu/swift/test/unit/obj/test_reconstructor.py\", line 2560, in test_reconstruct_fa_no_errors\n        self.assertIn(\u0027X-Backend-Fragment-Preferences\u0027, called_header)\n    AssertionError: \u0027X-Backend-Fragment-Preferences\u0027 not found in {\u0027Content-Length\u0027: \u00270\u0027, \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027, \u0027User-Agent\u0027: \u0027obj-reconstructor 2246\u0027}","commit_id":"6574ce31eeda1bdffa2534f21eed7ab0d62bce97"}]}