)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":14,"context_line":"offset timestamps."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Related-Bug: #2143206"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: I3a46d840553f6f9bf627b609432de6694bda15c9"},{"line_number":19,"context_line":"Signed-off-by: Jianjian Huo \u003cjhuo@nvidia.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"bc8f51d5_e9175577","line":17,"updated":"2026-03-12 15:37:07.000000000","message":"Related-Change: I946636249c61897b82e18451bf7a116261d190ab","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":14,"context_line":"offset timestamps."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Related-Bug: #2143206"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: I3a46d840553f6f9bf627b609432de6694bda15c9"},{"line_number":19,"context_line":"Signed-off-by: Jianjian Huo \u003cjhuo@nvidia.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"33b2cce7_4e9c65b1","line":17,"in_reply_to":"bc8f51d5_e9175577","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"46c72acd_845875f2","updated":"2026-03-17 22:51:45.000000000","message":"Thanks for the review and help!","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"ffb16bf7_4cda3870","updated":"2026-03-12 15:37:07.000000000","message":"This is similar to the recently added test_rebuild_with_non_durable_newer_data except this new test will fail now if there is a regression, whereas the previously added test wouldn\u0027t fail until Timestamps have jitter. That\u0027s good!\n\nI think the test can take advantage of some helper methods we have and therefore be more succinct. See https://review.opendev.org/c/openstack/swift/+/980310\n\nI also have some suggestions for the commenting (again, sorry!).","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"053f5f3f_fe5dba3f","updated":"2026-03-24 21:37:19.000000000","message":"Interestingly for Related-Change: I946636249c61897b82e18451bf7a116261d190ab when I revert the change the probetest doesn\u0027t fail:\n\n```\npytest swift/test/probe/test_reconstructor_rebuild.py::TestReconstructorRebuild -k test_rebuild_with_non_durable_newer_data -vsx\n```\n\n... but this one does!\n\n```\npytest swift/test/probe/test_reconstructor_rebuild.py::TestReconstructorRebuildReconcilerOffset -k test_rebuild_reconciled_object_with_offset_timestamp\n```\n\nI think a regression preventing probe test on master has GOT to make swift better.  But it might still benefit from some additional maintenance investment:\n\n982014: test: use node_key to track EC fragment responses | https://review.opendev.org/c/openstack/swift/+/982014","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"}],"test/probe/test_reconstructor_rebuild.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":650,"context_line":"    created by the reconciler when moving objects between storage policies."},{"line_number":651,"context_line":"    \"\"\""},{"line_number":652,"context_line":""},{"line_number":653,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":654,"context_line":"    @unittest.skipIf(not POLICIES_BY_TYPE.get(EC_POLICY),"},{"line_number":655,"context_line":"                     \"Need an EC policy\")"},{"line_number":656,"context_line":"    def setUp(self):"}],"source_content_type":"text/x-python","patch_set":1,"id":"24726af0_7afe5245","line":653,"updated":"2026-03-12 15:37:07.000000000","message":"in conjunction with the comment on the next line, this check will be unnecessary since an ECProbeTest will check for an ec policy and line 662 will check for a repl policy","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":650,"context_line":"    created by the reconciler when moving objects between storage policies."},{"line_number":651,"context_line":"    \"\"\""},{"line_number":652,"context_line":""},{"line_number":653,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":654,"context_line":"    @unittest.skipIf(not POLICIES_BY_TYPE.get(EC_POLICY),"},{"line_number":655,"context_line":"                     \"Need an EC policy\")"},{"line_number":656,"context_line":"    def setUp(self):"}],"source_content_type":"text/x-python","patch_set":1,"id":"19cf82f6_00595619","line":653,"in_reply_to":"24726af0_7afe5245","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":652,"context_line":""},{"line_number":653,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":654,"context_line":"    @unittest.skipIf(not POLICIES_BY_TYPE.get(EC_POLICY),"},{"line_number":655,"context_line":"                     \"Need an EC policy\")"},{"line_number":656,"context_line":"    def setUp(self):"},{"line_number":657,"context_line":"        super(TestReconstructorRebuildReconcilerOffset, self).setUp()"},{"line_number":658,"context_line":"        # identify the replication policy (not EC)"}],"source_content_type":"text/x-python","patch_set":1,"id":"32cf38b8_651e2f88","line":655,"updated":"2026-03-12 15:37:07.000000000","message":"this should already be taken care of by the ECProbeTest policy_requirements\n\nsee https://github.com/openstack/swift/blob/ef8b2a5f32adcc99fcbe9e377da90d081d3f7469/test/probe/common.py#L388","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":652,"context_line":""},{"line_number":653,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":654,"context_line":"    @unittest.skipIf(not POLICIES_BY_TYPE.get(EC_POLICY),"},{"line_number":655,"context_line":"                     \"Need an EC policy\")"},{"line_number":656,"context_line":"    def setUp(self):"},{"line_number":657,"context_line":"        super(TestReconstructorRebuildReconcilerOffset, self).setUp()"},{"line_number":658,"context_line":"        # identify the replication policy (not EC)"}],"source_content_type":"text/x-python","patch_set":1,"id":"93327ce5_6e10e558","line":655,"in_reply_to":"32cf38b8_651e2f88","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":711,"context_line":"                self.container_name.decode(\u0027utf8\u0027))"},{"line_number":712,"context_line":"            found_policy_indexes.add("},{"line_number":713,"context_line":"                metadata[\u0027X-Backend-Storage-Policy-Index\u0027])"},{"line_number":714,"context_line":"        self.assertGreater("},{"line_number":715,"context_line":"            len(found_policy_indexes), 1,"},{"line_number":716,"context_line":"            \u0027Container nodes should disagree about policy, got %r\u0027"},{"line_number":717,"context_line":"            % found_policy_indexes)"}],"source_content_type":"text/x-python","patch_set":1,"id":"9ddb36d9_9af7cf42","line":714,"updated":"2026-03-12 15:37:07.000000000","message":"why is this not exactly 2 policies?","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":711,"context_line":"                self.container_name.decode(\u0027utf8\u0027))"},{"line_number":712,"context_line":"            found_policy_indexes.add("},{"line_number":713,"context_line":"                metadata[\u0027X-Backend-Storage-Policy-Index\u0027])"},{"line_number":714,"context_line":"        self.assertGreater("},{"line_number":715,"context_line":"            len(found_policy_indexes), 1,"},{"line_number":716,"context_line":"            \u0027Container nodes should disagree about policy, got %r\u0027"},{"line_number":717,"context_line":"            % found_policy_indexes)"}],"source_content_type":"text/x-python","patch_set":1,"id":"f9451688_cded5591","line":714,"in_reply_to":"9ddb36d9_9af7cf42","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":831,"context_line":"                         md5(body, usedforsecurity\u003dFalse).hexdigest())"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"    def test_rebuild_reconciled_object_with_offset_timestamp(self):"},{"line_number":834,"context_line":"        # This test case exercises the bug fixed in previous commit that"},{"line_number":835,"context_line":"        # changed frag_prefs from using Timestamp.normal to Timestamp.internal."},{"line_number":836,"context_line":"        # When the reconciler moves an object from a replication policy to an"},{"line_number":837,"context_line":"        # EC policy, it adds a timestamp offset(offset\u003d3). If the reconstructor"}],"source_content_type":"text/x-python","patch_set":1,"id":"ee35db5f_c4b47ce6","line":834,"range":{"start_line":834,"start_character":52,"end_line":834,"end_character":67},"updated":"2026-03-12 15:37:07.000000000","message":"which commit? better to reference the launchpad bug\n\n```This test exercises the fix for https://bugs.launchpad.net/swift/+bug/2143206\n```","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":831,"context_line":"                         md5(body, usedforsecurity\u003dFalse).hexdigest())"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"    def test_rebuild_reconciled_object_with_offset_timestamp(self):"},{"line_number":834,"context_line":"        # This test case exercises the bug fixed in previous commit that"},{"line_number":835,"context_line":"        # changed frag_prefs from using Timestamp.normal to Timestamp.internal."},{"line_number":836,"context_line":"        # When the reconciler moves an object from a replication policy to an"},{"line_number":837,"context_line":"        # EC policy, it adds a timestamp offset(offset\u003d3). If the reconstructor"}],"source_content_type":"text/x-python","patch_set":1,"id":"263559b3_71d84039","line":834,"range":{"start_line":834,"start_character":52,"end_line":834,"end_character":67},"in_reply_to":"ee35db5f_c4b47ce6","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":834,"context_line":"        # This test case exercises the bug fixed in previous commit that"},{"line_number":835,"context_line":"        # changed frag_prefs from using Timestamp.normal to Timestamp.internal."},{"line_number":836,"context_line":"        # When the reconciler moves an object from a replication policy to an"},{"line_number":837,"context_line":"        # EC policy, it adds a timestamp offset(offset\u003d3). If the reconstructor"},{"line_number":838,"context_line":"        # sends the normal format(which strips the offset) in fragment"},{"line_number":839,"context_line":"        # preferences, peer nodes won\u0027t match it and may return wrong"},{"line_number":840,"context_line":"        # fragments. See https://bugs.launchpad.net/swift/+bug/2143206"},{"line_number":841,"context_line":"        self._setup_reconciled_ec_object()"}],"source_content_type":"text/x-python","patch_set":1,"id":"4fe615ab_429ce52e","line":838,"range":{"start_line":837,"start_character":59,"end_line":838,"end_character":58},"updated":"2026-03-12 15:37:07.000000000","message":"but it can\u0027t because we have fixed the bug 😊\n\nI\u0027d prefer it if we describe what the reconstructor should be doing rather than suggest that it might do something else:\n\n```\nThe test verifies that the reconstructor sends the internal timestamp\nformat (which includes the offset) in fragment preferences, so that\npeer nodes match it and return the correct fragments.\n```","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":834,"context_line":"        # This test case exercises the bug fixed in previous commit that"},{"line_number":835,"context_line":"        # changed frag_prefs from using Timestamp.normal to Timestamp.internal."},{"line_number":836,"context_line":"        # When the reconciler moves an object from a replication policy to an"},{"line_number":837,"context_line":"        # EC policy, it adds a timestamp offset(offset\u003d3). If the reconstructor"},{"line_number":838,"context_line":"        # sends the normal format(which strips the offset) in fragment"},{"line_number":839,"context_line":"        # preferences, peer nodes won\u0027t match it and may return wrong"},{"line_number":840,"context_line":"        # fragments. See https://bugs.launchpad.net/swift/+bug/2143206"},{"line_number":841,"context_line":"        self._setup_reconciled_ec_object()"}],"source_content_type":"text/x-python","patch_set":1,"id":"853553a5_d37479b9","line":838,"range":{"start_line":837,"start_character":59,"end_line":838,"end_character":58},"in_reply_to":"4fe615ab_429ce52e","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":844,"context_line":"        # This is critical for exposing the bug: if the reconstructor sends"},{"line_number":845,"context_line":"        # the wrong (normal) timestamp in frag_prefs, peers won\u0027t match"},{"line_number":846,"context_line":"        # the offset timestamp and will fall back to returning \"newest\""},{"line_number":847,"context_line":"        # (v2) fragments. v2 data has to be larger than v1 to ensure that"},{"line_number":848,"context_line":"        # the reconstructor will use v2 data to rebuild the fragment."},{"line_number":849,"context_line":"        v2_data \u003d b\u0027newer-non-durable-version\u0027 * 2000"},{"line_number":850,"context_line":"        internal_client \u003d self.make_internal_client()"},{"line_number":851,"context_line":"        v2_body \u003d ProbeBody(total\u003dlen(v2_data))"}],"source_content_type":"text/x-python","patch_set":1,"id":"76f677ae_e2170d18","line":848,"range":{"start_line":847,"start_character":26,"end_line":848,"end_character":69},"updated":"2026-03-12 15:37:07.000000000","message":"This sentence *alone* makes a false statement. I understand that it\u0027s trying to point out that if the bug had not been fixed then we\u0027d need to have v2 size to be larger (or equal to in fact) than v1 data (and also newer) for the test to fail. But, as above, it reads a little too much like a statement of how the reconstructor works.\n\nI think it\u0027s tricky to write a comment about how things *used to be*....perhaps easier to describe how things should be now:\n\n```\n# PUT a newer non-durable version (let\u0027s call it v2) on all EC nodes.\n# This is critical for detecting a regression to the bug fix:\n# v2 frags must be newer and must have at least as many bytes as v1 frags.\n```","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":844,"context_line":"        # This is critical for exposing the bug: if the reconstructor sends"},{"line_number":845,"context_line":"        # the wrong (normal) timestamp in frag_prefs, peers won\u0027t match"},{"line_number":846,"context_line":"        # the offset timestamp and will fall back to returning \"newest\""},{"line_number":847,"context_line":"        # (v2) fragments. v2 data has to be larger than v1 to ensure that"},{"line_number":848,"context_line":"        # the reconstructor will use v2 data to rebuild the fragment."},{"line_number":849,"context_line":"        v2_data \u003d b\u0027newer-non-durable-version\u0027 * 2000"},{"line_number":850,"context_line":"        internal_client \u003d self.make_internal_client()"},{"line_number":851,"context_line":"        v2_body \u003d ProbeBody(total\u003dlen(v2_data))"}],"source_content_type":"text/x-python","patch_set":1,"id":"ffee0a7c_1dc86688","line":848,"range":{"start_line":847,"start_character":26,"end_line":848,"end_character":69},"in_reply_to":"76f677ae_e2170d18","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":863,"context_line":"                _hdrs, durable_frag_etag \u003d self.direct_get("},{"line_number":864,"context_line":"                    node, self.ec_part)"},{"line_number":865,"context_line":"                self.assertEqual(self.frag_etags[node[\u0027index\u0027]],"},{"line_number":866,"context_line":"                                 durable_frag_etag)"},{"line_number":867,"context_line":"                _hdrs, newest_frag_etag \u003d self.direct_get("},{"line_number":868,"context_line":"                    node, self.ec_part, require_durable\u003dFalse)"},{"line_number":869,"context_line":"                self.assertNotEqual(durable_frag_etag, newest_frag_etag,"}],"source_content_type":"text/x-python","patch_set":1,"id":"37e5b7d5_9f97eaf5","line":866,"updated":"2026-03-12 15:37:07.000000000","message":"I try whenever possible to avoid assert inside for loops because when they fail you don\u0027t know if later iterations would also fail, and because you have to annotate the failure with something to record which iteration failed.\n\nI think a better pattern is to collect all results and then make assertion(s) on the collection.","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":863,"context_line":"                _hdrs, durable_frag_etag \u003d self.direct_get("},{"line_number":864,"context_line":"                    node, self.ec_part)"},{"line_number":865,"context_line":"                self.assertEqual(self.frag_etags[node[\u0027index\u0027]],"},{"line_number":866,"context_line":"                                 durable_frag_etag)"},{"line_number":867,"context_line":"                _hdrs, newest_frag_etag \u003d self.direct_get("},{"line_number":868,"context_line":"                    node, self.ec_part, require_durable\u003dFalse)"},{"line_number":869,"context_line":"                self.assertNotEqual(durable_frag_etag, newest_frag_etag,"}],"source_content_type":"text/x-python","patch_set":1,"id":"25158a9e_120bc8f6","line":866,"in_reply_to":"37e5b7d5_9f97eaf5","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":871,"context_line":"                                    \u0027differ from durable v1 frag on \u0027"},{"line_number":872,"context_line":"                                    \u0027node %s\u0027 % node[\u0027device\u0027])"},{"line_number":873,"context_line":"            except direct_client.DirectClientException:"},{"line_number":874,"context_line":"                pass  # some nodes may not have fragments"},{"line_number":875,"context_line":""},{"line_number":876,"context_line":"        # Delete one EC fragment so it needs rebuilding"},{"line_number":877,"context_line":"        fail_index \u003d list(self.frag_etags.keys())[0]"}],"source_content_type":"text/x-python","patch_set":1,"id":"afc7eece_621ca24b","line":874,"updated":"2026-03-12 15:37:07.000000000","message":"the try/except should wrap each direct_get individually so that we\u0027d detect a device that maybe only has the second frag","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":871,"context_line":"                                    \u0027differ from durable v1 frag on \u0027"},{"line_number":872,"context_line":"                                    \u0027node %s\u0027 % node[\u0027device\u0027])"},{"line_number":873,"context_line":"            except direct_client.DirectClientException:"},{"line_number":874,"context_line":"                pass  # some nodes may not have fragments"},{"line_number":875,"context_line":""},{"line_number":876,"context_line":"        # Delete one EC fragment so it needs rebuilding"},{"line_number":877,"context_line":"        fail_index \u003d list(self.frag_etags.keys())[0]"}],"source_content_type":"text/x-python","patch_set":1,"id":"c4fa598f_4f4f15ff","line":874,"in_reply_to":"afc7eece_621ca24b","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":883,"context_line":"        self.break_nodes(self.ec_nodes, self.ec_part, [fail_list_idx], [])"},{"line_number":884,"context_line":""},{"line_number":885,"context_line":"        # Verify the fragment is gone"},{"line_number":886,"context_line":"        with self.assertRaises(direct_client.DirectClientException) as cm:"},{"line_number":887,"context_line":"            direct_client.direct_get_object("},{"line_number":888,"context_line":"                fail_node, self.ec_part, self.account,"},{"line_number":889,"context_line":"                self.container_name.decode(\u0027utf8\u0027),"}],"source_content_type":"text/x-python","patch_set":1,"id":"898e1450_e246ae09","line":886,"updated":"2026-03-12 15:37:07.000000000","message":"``self.assert_direct_get_fails((fail_node, self.ec_part, 404)`` will do this","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":883,"context_line":"        self.break_nodes(self.ec_nodes, self.ec_part, [fail_list_idx], [])"},{"line_number":884,"context_line":""},{"line_number":885,"context_line":"        # Verify the fragment is gone"},{"line_number":886,"context_line":"        with self.assertRaises(direct_client.DirectClientException) as cm:"},{"line_number":887,"context_line":"            direct_client.direct_get_object("},{"line_number":888,"context_line":"                fail_node, self.ec_part, self.account,"},{"line_number":889,"context_line":"                self.container_name.decode(\u0027utf8\u0027),"}],"source_content_type":"text/x-python","patch_set":1,"id":"346e0b35_a1a770b7","line":886,"in_reply_to":"898e1450_e246ae09","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":906,"context_line":"        hasher \u003d md5(usedforsecurity\u003dFalse)"},{"line_number":907,"context_line":"        for chunk in rebuilt_data:"},{"line_number":908,"context_line":"            hasher.update(chunk)"},{"line_number":909,"context_line":"        rebuilt_frag_etag \u003d hasher.hexdigest()"},{"line_number":910,"context_line":""},{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"e52f3368_0f7794ce","line":909,"updated":"2026-03-12 15:37:07.000000000","message":"``rebuilt_hdrs, rebuilt_frag_etag \u003d self.direct_get(\n            fail_node, self.ec_part)``  does the above 10 lines","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":906,"context_line":"        hasher \u003d md5(usedforsecurity\u003dFalse)"},{"line_number":907,"context_line":"        for chunk in rebuilt_data:"},{"line_number":908,"context_line":"            hasher.update(chunk)"},{"line_number":909,"context_line":"        rebuilt_frag_etag \u003d hasher.hexdigest()"},{"line_number":910,"context_line":""},{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"9d3122f8_88ed90ac","line":909,"in_reply_to":"e52f3368_0f7794ce","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":908,"context_line":"            hasher.update(chunk)"},{"line_number":909,"context_line":"        rebuilt_frag_etag \u003d hasher.hexdigest()"},{"line_number":910,"context_line":""},{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"},{"line_number":913,"context_line":"        # \"1234567890.12345_0000000000000003\" on peer nodes."},{"line_number":914,"context_line":"        self.assertEqual(orig_frag_etag, rebuilt_frag_etag,"}],"source_content_type":"text/x-python","patch_set":1,"id":"70bb80e0_a79956c8","line":911,"updated":"2026-03-12 15:37:07.000000000","message":"it will be more useful to us in future to just have the launchpad link","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":908,"context_line":"            hasher.update(chunk)"},{"line_number":909,"context_line":"        rebuilt_frag_etag \u003d hasher.hexdigest()"},{"line_number":910,"context_line":""},{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"},{"line_number":913,"context_line":"        # \"1234567890.12345_0000000000000003\" on peer nodes."},{"line_number":914,"context_line":"        self.assertEqual(orig_frag_etag, rebuilt_frag_etag,"}],"source_content_type":"text/x-python","patch_set":1,"id":"954484fd_c444af7d","line":911,"in_reply_to":"70bb80e0_a79956c8","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"efc39fce828b3cda5b697486f012b794667a345d","unresolved":true,"context_lines":[{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"},{"line_number":913,"context_line":"        # \"1234567890.12345_0000000000000003\" on peer nodes."},{"line_number":914,"context_line":"        self.assertEqual(orig_frag_etag, rebuilt_frag_etag,"},{"line_number":915,"context_line":"                         \u0027Rebuilt fragment does not match original v1 \u0027"},{"line_number":916,"context_line":"                         \u0027fragment; reconstruction may have incorrectly \u0027"},{"line_number":917,"context_line":"                         \u0027used newer non-durable v2 data due to wrong \u0027"}],"source_content_type":"text/x-python","patch_set":1,"id":"16e7ff86_20a617f3","line":914,"updated":"2026-03-10 23:22:35.000000000","message":"I rolled back previous reconstructor fix (978572: obj-reconstructor: use Timestamp.internal in frag prefs | https://review.opendev.org/c/openstack/swift/+/978572), and then I saw this test failure:\n\n```\n\u003e       self.assertEqual(orig_frag_etag, rebuilt_frag_etag,\n                         \u0027Rebuilt fragment does not match original v1 \u0027\n                         \u0027fragment; reconstruction may have incorrectly \u0027\n                         \u0027used newer non-durable v2 data due to wrong \u0027\n                         \u0027timestamp format in fragment preferences\u0027)\nE       AssertionError: \u0027fb94d3ed9f0f2da33b054fc2e7107e39\u0027 !\u003d \u002798e3b854eb097092d4321825d09da666\u0027\nE       - fb94d3ed9f0f2da33b054fc2e7107e39\nE       + 98e3b854eb097092d4321825d09da666\nE        : Rebuilt fragment does not match original v1 fragment; reconstruction may have incorrectly used newer non-durable v2 data due to wrong timestamp format in fragment preferences\n\ntest/probe/test_reconstructor_rebuild.py:909: AssertionError\n```","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":911,"context_line":"        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp"},{"line_number":912,"context_line":"        # \"1234567890.12345\" won\u0027t match the actual offset timestamp"},{"line_number":913,"context_line":"        # \"1234567890.12345_0000000000000003\" on peer nodes."},{"line_number":914,"context_line":"        self.assertEqual(orig_frag_etag, rebuilt_frag_etag,"},{"line_number":915,"context_line":"                         \u0027Rebuilt fragment does not match original v1 \u0027"},{"line_number":916,"context_line":"                         \u0027fragment; reconstruction may have incorrectly \u0027"},{"line_number":917,"context_line":"                         \u0027used newer non-durable v2 data due to wrong \u0027"}],"source_content_type":"text/x-python","patch_set":1,"id":"e75bbd46_55662bf8","line":914,"in_reply_to":"16e7ff86_20a617f3","updated":"2026-03-17 22:51:45.000000000","message":"Done","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9b2c2c99e071f43ba32d36e66f9fabdbeaa5c554","unresolved":true,"context_lines":[{"line_number":937,"context_line":"            body_parts.append(chunk)"},{"line_number":938,"context_line":"        body \u003d b\u0027\u0027.join(body_parts)"},{"line_number":939,"context_line":"        self.assertEqual(self.obj_etag,"},{"line_number":940,"context_line":"                         md5(body, usedforsecurity\u003dFalse).hexdigest())"},{"line_number":941,"context_line":""},{"line_number":942,"context_line":""},{"line_number":943,"context_line":"class TestReconstructorRebuildUTF8(TestReconstructorRebuild):"}],"source_content_type":"text/x-python","patch_set":1,"id":"659a1885_0afe2e61","line":940,"updated":"2026-03-12 15:37:07.000000000","message":"self.proxy_get() does this request and etag calculation","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"33dacc93f3d4aa5cf069fd87a5ecadbf8595a8b6","unresolved":false,"context_lines":[{"line_number":937,"context_line":"            body_parts.append(chunk)"},{"line_number":938,"context_line":"        body \u003d b\u0027\u0027.join(body_parts)"},{"line_number":939,"context_line":"        self.assertEqual(self.obj_etag,"},{"line_number":940,"context_line":"                         md5(body, usedforsecurity\u003dFalse).hexdigest())"},{"line_number":941,"context_line":""},{"line_number":942,"context_line":""},{"line_number":943,"context_line":"class TestReconstructorRebuildUTF8(TestReconstructorRebuild):"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f957089_a5d5916e","line":940,"in_reply_to":"659a1885_0afe2e61","updated":"2026-03-17 22:51:45.000000000","message":"Acknowledged","commit_id":"52f077e42355fdce19c3bd2e468376c8458aa037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":658,"context_line":"                         if p.policy_type !\u003d EC_POLICY]"},{"line_number":659,"context_line":"        if not repl_policies:"},{"line_number":660,"context_line":"            self.skipTest(\"Need a replication policy\")"},{"line_number":661,"context_line":"        self.repl_policy \u003d repl_policies[0]"},{"line_number":662,"context_line":"        self.ec_policy \u003d self.policy"},{"line_number":663,"context_line":"        self.brain \u003d BrainSplitter(self.url, self.token,"},{"line_number":664,"context_line":"                                   self.container_name.decode(\u0027utf8\u0027),"}],"source_content_type":"text/x-python","patch_set":3,"id":"ea59399c_c5315042","line":661,"updated":"2026-03-24 21:37:19.000000000","message":"does the \"repl-ness\" of this policy matter to test?\n\nIf `self.other_policy \u003d random.choice([p for p in self.policies if p !\u003d self.policy])` works that might be better.","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":710,"context_line":"            found_policy_indexes.add("},{"line_number":711,"context_line":"                metadata[\u0027X-Backend-Storage-Policy-Index\u0027])"},{"line_number":712,"context_line":"        self.assertEqual("},{"line_number":713,"context_line":"            len(found_policy_indexes), 2,"},{"line_number":714,"context_line":"            \u0027Container nodes should disagree about policy, got %r\u0027"},{"line_number":715,"context_line":"            % found_policy_indexes)"},{"line_number":716,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"6b4be118_22c4e61e","line":713,"updated":"2026-03-24 21:37:19.000000000","message":"I\u0027d prefer this as as a Counter\n\n```\n        found_policy_indexes \u003d {}\n        for node in container_nodes:\n            node_key \u003d lambda n: (n[\u0027ip\u0027], n[\u0027port\u0027], n[\u0027device\u0027])\n            metadata \u003d direct_client.direct_head_container(\n                node, container_part, self.account,\n                self.container_name.decode(\u0027utf8\u0027))\n            found_policy_indexes[node_key(node)] \u003d int(\n                metadata[\u0027X-Backend-Storage-Policy-Index\u0027])\n        self.assertEqual(\n            Counter(found_policy_indexes.values()), {\n                int(self.policy): 2,\n                int(self.other_policy): 2,\n            },\n            \u0027Container nodes should disagree about policy, got %r\u0027\n            % found_policy_indexes)\n```\n\nI think it fails nicer:\n\n```\nE   AssertionError: Counter({0: 2, 1: 1}) !\u003d {1: 2, 0: 2} : Container nodes should disagree about policy, got {(\u0027127.0.0.2\u0027, 6021, \u0027sdb2\u0027): 0, (\u0027127.0.0.4\u0027, 6041, \u0027sdb4\u0027): 1, (\u0027127.0.0.1\u0027, 6011, \u0027sdb1\u0027): 0}\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":732,"context_line":"                found_on_repl \u003d True"},{"line_number":733,"context_line":"                break"},{"line_number":734,"context_line":"            except direct_client.ClientException:"},{"line_number":735,"context_line":"                continue"},{"line_number":736,"context_line":"        self.assertTrue(found_on_repl,"},{"line_number":737,"context_line":"                        \u0027Object not found on replication policy nodes\u0027)"},{"line_number":738,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"98961299_b6baca08","line":735,"updated":"2026-03-24 21:37:19.000000000","message":"this is maybe a premature optimization; I don\u0027t think there\u0027s any inconsistency in the data layer at this point:\n\n```\n(Pdb) Counter(resp[\u0027Etag\u0027] for resp in found_on_other.values())\nCounter({\u0027\"c617476bfbbcd29ec30b0686e9847938\"\u0027: 3})\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":776,"context_line":"                self.frag_headers[node[\u0027index\u0027]] \u003d hdrs"},{"line_number":777,"context_line":"            except direct_client.ClientException as err:"},{"line_number":778,"context_line":"                if err.http_status !\u003d HTTP_NOT_FOUND:"},{"line_number":779,"context_line":"                    raise"},{"line_number":780,"context_line":""},{"line_number":781,"context_line":"        # we need at least ec_ndata fragments for reconstruction"},{"line_number":782,"context_line":"        self.assertGreaterEqual("}],"source_content_type":"text/x-python","patch_set":3,"id":"bb0bb1e0_5a9f8d5d","line":779,"updated":"2026-03-24 21:37:19.000000000","message":"sketch\n\nbetter I think if tests where we have full control over the system don\u0027t tolerate errors we don\u0027t expect:\n\n```\n         self.frag_etags \u003d {}\n         self.frag_headers \u003d {}\n+        frag_failures \u003d {}\n         for node in self.ec_nodes:\n             try:\n                 hdrs, etag \u003d self.direct_get(node, self.ec_part)\n                 self.frag_etags[node[\u0027index\u0027]] \u003d etag\n                 self.frag_headers[node[\u0027index\u0027]] \u003d hdrs\n             except direct_client.ClientException as err:\n-                if err.http_status !\u003d HTTP_NOT_FOUND:\n-                    raise\n+                frag_failures[_format_node(node)] \u003d err\n+        if frag_failures:\n+            self.fail(\u0027\\n\u0027.join([\n+                \u0027    Node %r raised %r\u0027 % (node, exc)\n+                for (node, exc) in frag_failures.items\n+            ]))\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":787,"context_line":""},{"line_number":788,"context_line":"        # Verify the timestamp has an offset (added by reconciler)"},{"line_number":789,"context_line":"        sample_ts_str \u003d list("},{"line_number":790,"context_line":"            self.frag_headers.values())[0][\u0027X-Backend-Timestamp\u0027]"},{"line_number":791,"context_line":"        self.obj_timestamp \u003d Timestamp(sample_ts_str)"},{"line_number":792,"context_line":"        self.assertGreater("},{"line_number":793,"context_line":"            self.obj_timestamp.offset, 0,"}],"source_content_type":"text/x-python","patch_set":3,"id":"0942f61f_de78913d","line":790,"updated":"2026-03-24 21:37:19.000000000","message":"slightly stronger I think if we assert all of these responses have the *same* x-backend-timestamp before we pop it off the front of the list.","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":796,"context_line":""},{"line_number":797,"context_line":"        # Verify proxy GET returns correct data (may need retries due to"},{"line_number":798,"context_line":"        # cached container info in the proxy)"},{"line_number":799,"context_line":"        timeout \u003d time.time() + 60"},{"line_number":800,"context_line":"        while True:"},{"line_number":801,"context_line":"            try:"},{"line_number":802,"context_line":"                headers, etag \u003d self.proxy_get()"}],"source_content_type":"text/x-python","patch_set":3,"id":"59833b98_c72e0396","line":799,"updated":"2026-03-24 21:37:19.000000000","message":"you can clear cached container info with a container metadata update/post","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":825,"context_line":"        # PUT a newer non-durable version (let\u0027s call it v2) on all EC nodes."},{"line_number":826,"context_line":"        # This is critical for detecting a regression to the bug fix:"},{"line_number":827,"context_line":"        # v2 frags must be newer and must have at least as many bytes as v1"},{"line_number":828,"context_line":"        # frags."},{"line_number":829,"context_line":"        v2_data \u003d b\u0027newer-non-durable-version\u0027 * 2000"},{"line_number":830,"context_line":"        internal_client \u003d self.make_internal_client()"},{"line_number":831,"context_line":"        v2_body \u003d ProbeBody(total\u003dlen(v2_data))"}],"source_content_type":"text/x-python","patch_set":3,"id":"aa0ed605_958a7f03","line":828,"updated":"2026-03-24 21:37:19.000000000","message":"\u003e at least as many bytes as [the original] frags.\n\ninteresting!  So if we do a PUT w/ Content-Length: old_metadata and write a smaller number of bytes we just break HTTP framing and get a Timeout!","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":828,"context_line":"        # frags."},{"line_number":829,"context_line":"        v2_data \u003d b\u0027newer-non-durable-version\u0027 * 2000"},{"line_number":830,"context_line":"        internal_client \u003d self.make_internal_client()"},{"line_number":831,"context_line":"        v2_body \u003d ProbeBody(total\u003dlen(v2_data))"},{"line_number":832,"context_line":"        internal_client.upload_object("},{"line_number":833,"context_line":"            v2_body, self.account,"},{"line_number":834,"context_line":"            self.container_name.decode(\u0027utf8\u0027),"}],"source_content_type":"text/x-python","patch_set":3,"id":"6e9994e6_9ff8c491","line":831,"updated":"2026-03-24 21:37:19.000000000","message":"this seems like a *really* dumb way to come up this number:\n\n```\n\u003e\u003e\u003e len(b\u0027newer-non-durable-version\u0027 * 2000)\n50000\n```\n\nI mean it\u0027d be one thing if we *used* v2_data?\n\n```\n@@ -824,11 +843,12 @@ class TestReconstructorRebuildReconcilerOffset(ECProbeTest):\n \n         # PUT a newer non-durable version (let\u0027s call it v2) on all EC nodes.\n         # This is critical for detecting a regression to the bug fix:\n-        # v2 frags must be newer and must have at least as many bytes as v1\n-        # frags.\n-        v2_data \u003d b\u0027newer-non-durable-version\u0027 * 2000\n+        # v2 frags must be newer and must have at least as many bytes as the\n+        # original frags.\n         internal_client \u003d self.make_internal_client()\n-        v2_body \u003d ProbeBody(total\u003dlen(v2_data))\n+        orig_headers \u003d random.choice(list(self.frag_headers.values()))\n+        orig_length \u003d int(orig_headers[\u0027X-Object-Sysmeta-Ec-Content-Length\u0027])\n+        v2_body \u003d ProbeBody(total\u003d2 * orig_length)\n         internal_client.upload_object(\n             v2_body, self.account,\n             self.container_name.decode(\u0027utf8\u0027),\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":839,"context_line":"        dev_frag_etags \u003d defaultdict(list)"},{"line_number":840,"context_line":"        for node in self.ec_nodes:"},{"line_number":841,"context_line":"            if node[\u0027index\u0027] not in self.frag_etags:"},{"line_number":842,"context_line":"                continue"},{"line_number":843,"context_line":"            try:"},{"line_number":844,"context_line":"                etag \u003d self.direct_get(node, self.ec_part)[1]"},{"line_number":845,"context_line":"                self.assertEqual(self.frag_etags[node[\u0027index\u0027]], etag)"}],"source_content_type":"text/x-python","patch_set":3,"id":"10dec078_7ee4fe4d","line":842,"updated":"2026-03-24 21:37:19.000000000","message":"sketch","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":862,"context_line":"        fail_index \u003d list(self.frag_etags.keys())[0]"},{"line_number":863,"context_line":"        fail_node \u003d [n for n in self.ec_nodes"},{"line_number":864,"context_line":"                     if n[\u0027index\u0027] \u003d\u003d fail_index][0]"},{"line_number":865,"context_line":"        orig_frag_etag \u003d self.frag_etags[fail_index]"},{"line_number":866,"context_line":""},{"line_number":867,"context_line":"        fail_list_idx \u003d self.ec_nodes.index(fail_node)"},{"line_number":868,"context_line":"        self.break_nodes(self.ec_nodes, self.ec_part, [fail_list_idx], [])"}],"source_content_type":"text/x-python","patch_set":3,"id":"c97b5a8f_ccc96121","line":865,"updated":"2026-03-24 21:37:19.000000000","message":"there\u0027s a weird circular reasoning here\n\nwe pick the index of the first etag\nthen we find the node with that index\nthen we lookup the etag for that index\n\nDoes the test care if we pick a random ec_node to fail?\n\n```\n         # Delete one EC fragment so it needs rebuilding\n-        fail_index \u003d list(self.frag_etags.keys())[0]\n-        fail_node \u003d [n for n in self.ec_nodes\n-                     if n[\u0027index\u0027] \u003d\u003d fail_index][0]\n-        orig_frag_etag \u003d self.frag_etags[fail_index]\n+        fail_node \u003d random.choice(self.ec_nodes)\n+        orig_frag_etag \u003d self.frag_etags[_format_node(fail_node)]\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":884,"context_line":"                         \u0027Rebuilt fragment does not match original v1 \u0027"},{"line_number":885,"context_line":"                         \u0027fragment; reconstruction may have incorrectly \u0027"},{"line_number":886,"context_line":"                         \u0027used newer non-durable v2 data due to wrong \u0027"},{"line_number":887,"context_line":"                         \u0027timestamp format in fragment preferences\u0027)"},{"line_number":888,"context_line":""},{"line_number":889,"context_line":"        # Verify the rebuilt fragment has the correct offset timestamp"},{"line_number":890,"context_line":"        rebuilt_ts \u003d Timestamp(rebuilt_hdrs[\u0027X-Backend-Timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"b11d3b28_b54d7654","line":887,"updated":"2026-03-24 21:37:19.000000000","message":"when I \"revert\" the \"fix\"\n\n```\n(vagrant-swift-all-in-one) cgerrard@NVStation:~/Workspace/vagrant-swift-all-in-one/swift$ git diff -R 1f488b2842eb27c9b1caad44caffb18f8f95a228^ -- swift/obj/reconstructor.py\ndiff --git b/swift/obj/reconstructor.py a/swift/obj/reconstructor.py\nindex 882f5e9267..8d7d90f837 100644\n--- b/swift/obj/reconstructor.py\n+++ a/swift/obj/reconstructor.py\n@@ -551,7 +551,7 @@ class ObjectReconstructor(Daemon):\n         headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(policy)\n         headers[\u0027X-Backend-Replication\u0027] \u003d \u0027True\u0027\n         local_timestamp \u003d Timestamp(datafile_metadata[\u0027X-Timestamp\u0027])\n-        frag_prefs \u003d [{\u0027timestamp\u0027: local_timestamp.internal, \u0027exclude\u0027: []}]\n+        frag_prefs \u003d [{\u0027timestamp\u0027: local_timestamp.normal, \u0027exclude\u0027: []}]\n         headers[\u0027X-Backend-Fragment-Preferences\u0027] \u003d json.dumps(frag_prefs)\n         path \u003d datafile_metadata[\u0027name\u0027]\n \n(vagrant-swift-all-in-one) cgerrard@NVStation:~/Workspace/vagrant-swift-all-in-one/swift$ git diff -R 1f488b2842eb27c9b1caad44caffb18f8f95a228^ -- swift/obj/reconstructor.py | git apply\n```\n\nI get a failure like this:\n\n\n```\n        # With the prevous bug (Timestamp.normal in frag_prefs), the timestamp\n        # \"1234567890.12345\" won\u0027t match the actual offset timestamp\n        # \"1234567890.12345_0000000000000003\" on peer nodes.\n\u003e       self.assertEqual(orig_frag_etag, rebuilt_frag_etag,\n                         \u0027Rebuilt fragment does not match original v1 \u0027\n                         \u0027fragment; reconstruction may have incorrectly \u0027\n                         \u0027used newer non-durable v2 data due to wrong \u0027\n                         \u0027timestamp format in fragment preferences\u0027)\nE       AssertionError: \u0027fb94d3ed9f0f2da33b054fc2e7107e39\u0027 !\u003d \u00279af1856239759470e149b89528710a46\u0027\nE       - fb94d3ed9f0f2da33b054fc2e7107e39\nE       + 9af1856239759470e149b89528710a46\nE        : Rebuilt fragment does not match original v1 fragment; reconstruction may have incorrectly used newer non-durable v2 data due to wrong timestamp format in fragment preferences\n\nswift/test/probe/test_reconstructor_rebuild.py:883: AssertionError\n```","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"81ed396cd1cdebdcccb234b4a48bf9e2ae830628","unresolved":true,"context_lines":[{"line_number":894,"context_line":"                                          self.obj_timestamp.internal))"},{"line_number":895,"context_line":""},{"line_number":896,"context_line":"        # Verify the rebuilt fragment is durable"},{"line_number":897,"context_line":"        self.assertIn(\u0027X-Backend-Durable-Timestamp\u0027, rebuilt_hdrs)"},{"line_number":898,"context_line":""},{"line_number":899,"context_line":"        # Proxy GET should still return correct data"},{"line_number":900,"context_line":"        headers, actual_etag \u003d self.proxy_get()"}],"source_content_type":"text/x-python","patch_set":3,"id":"2e2128a7_8d6c0bac","line":897,"updated":"2026-03-24 21:37:19.000000000","message":"non-durable fragment responses return an x-backend-data-timestamp AND the latest x-backend-durable-timestamp (if available)\n\nJust because a response includes an x-backend-durable-timestmap doesn\u0027t mean that it\u0027s durable - it needs to also be \u003d\u003d x-backend-data-timestamp","commit_id":"4c492f5d486d8df374a1f69e21a3845c9d79468a"}]}