)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"},{"line_number":29,"context_line":"you were on master today (excpet the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: Ib4690e274e3d05d4da3ad1322e1809b56f60288b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"95236b5a_ad30f5b1","line":29,"updated":"2025-09-25 06:14:18.000000000","message":"s/excpet/except","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d93eea271079c69c4cb1da441d2bc4540d9a344f","unresolved":true,"context_lines":[{"line_number":13,"context_line":"return a 503 response.  Well behaved clients should try and re-upload a"},{"line_number":14,"context_line":"PUT that returns 503."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"This patch is an improvement in that we won\u0027t return a 201 to a client"},{"line_number":17,"context_line":"unless a durable and consistent frag set is written."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"But we\u0027ll need more work to ensure that a set of non-durable frags are"},{"line_number":20,"context_line":"guaranteed to be or become consistent.  Even with this change, it should"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"4b6e2647_e9582fa2","line":17,"range":{"start_line":16,"start_character":37,"end_line":17,"end_character":52},"updated":"2025-09-25 14:37:41.000000000","message":"repeating myself: the additional probe tests in https://review.opendev.org/c/openstack/swift/+/962003/3 show this isn\u0027t true: we can still return 201 to both clients and have inconsistent but durable frag (although there will at least be a quorum of consistent frags).","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":22,"context_line":"durables with the same timestamp - but we don\u0027t know how to write that"},{"line_number":23,"context_line":"test yet [1]."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"3b8dc6c4_7d48916c","line":25,"updated":"2025-10-01 21:53:27.000000000","message":"Do we intend to do anything about it? Maybe start with writing a bug for it? Or is it more of a \"note\" than a \"todo\"?\n\nI guess maybe we could get the object-server to return 409? Then if one of the clients gets a 201, the other should get a 409 instead of a 503, right? I think that\u0027ll also address some of Alistair\u0027s concerns.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"},{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"d53231e1_b16925a3","line":27,"range":{"start_line":27,"start_character":38,"end_line":27,"end_character":51},"updated":"2025-10-01 21:53:27.000000000","message":"Decidedly -- though I suppose fixing it is out of scope for the moment.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: Ib4690e274e3d05d4da3ad1322e1809b56f60288b"},{"line_number":33,"context_line":"Co-Authored-By: Jianjian Huo \u003cjhuo@nvidia.com\u003e"},{"line_number":34,"context_line":"Co-Authored-By: Alistair Coles \u003calistairncoles@gmail.com\u003e"},{"line_number":35,"context_line":"Signed-off-by: Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"2fa40a60_ecf6d69f","line":32,"updated":"2025-10-01 21:53:27.000000000","message":"Should we add a\n\n\u003e Partial-Bug: #1971686\n\n? And we still need to think about how to handle folks without `O_TMPFILE` support....","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a77ef01566adbe078560c2d7287869e4c018a855","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"74838202_f35c6568","updated":"2024-08-28 20:07:25.000000000","message":"When I was testing this again yesterday I felt like I was having trouble creating the mis-matched etag fragments with encryption and concurrent uploads.  I want to write a probetest that can more reliably reproduce the issue - but I haven\u0027t thought of a strategy that seems worth commiting to yet.","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"4834dc0f_fe8c9890","updated":"2025-02-05 16:09:01.000000000","message":"I know there\u0027s an issue going on here; but I think these tests could be better.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6983434069772330db7c088366042a374f14e386","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"edc327e3_ef0dfe4b","updated":"2025-08-12 22:42:48.000000000","message":"I should take a stab at improving these tests.  Although they do seem to work to recreate the issue if anyone else wanted to dig into and help.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"fdf69e06_8012732a","updated":"2025-09-19 14:20:21.000000000","message":"I pushed some changes addressing some (but not all) of my comments here https://review.opendev.org/c/openstack/swift/+/961802\n\nI may need some help understanding the zipper class, but the tests seem to \"work\".","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"2993ec83_f2b6fca6","updated":"2025-09-19 21:27:59.000000000","message":"I think many of these comments will be addressed in 961802: sq: test_timestamp_collision fixups | https://review.opendev.org/c/openstack/swift/+/961802 - which we should squash!","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c9244a53_009c8d7d","updated":"2025-09-23 15:23:02.000000000","message":"I have a follow-up that makes the probe test more predictable, and adds another test that creates a partial but durable frag set. https://review.opendev.org/c/openstack/swift/+/962003","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ac4dd29ecf20ae11cab397b5f3b39526e0d4c8de","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"f2dbba69_2dd79dc9","updated":"2025-09-24 10:10:31.000000000","message":"I\u0027ve added a test in my follow-up patch ``test_overlap_data_write_streams_both_succeed_one_durable_set`` that shows a scenario where we can still have both requests return 201 to the client but have mixed durable frags on disk:\n\n* req2 writes fragX\n* req1 fails to write fragX\n* req1 writes N-1 other frags\n* req2 fails to write fragY\n* req1 got N-1 ok frags so makes N-1 frags durable\n* req1 -\u003e 201 to client\n* req2 writes N-2 other frags over the durable frags!! \n* req 2 got N-2 plus fragX ok frags, i.e. N-1, so makes them durable\n* req2 -\u003e 201 to client\n\nThat does NOT mean this patch is not necessary, but it does mean that this patch cannot claim to always return 503 on a client timestamp collision.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"8cabf269_6c7a686b","updated":"2025-09-19 23:19:58.000000000","message":"we\u0027ll see if zuul likes it and we can keep writing some more tests","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ec0fe93878008e2ec57a842f04065431f2e77d2d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"b0ce7134_a3abf3e2","in_reply_to":"f2dbba69_2dd79dc9","updated":"2025-09-25 14:32:13.000000000","message":"\u003e we can still have both requests return 201 to the client but have mixed durable frags on disk\n\nthinking about this more, perhaps the obj server should return 409 (conflict) if it gets EEXIST while linking to the .data file. Then the proxy should NOT proceed to the commit phase if any frag returns 409. That would significantly further narrow the opportunity to reach a set of mixed-metadata durable frags. Typically both requests would get a 503 due to one or more backend 409s, and we\u0027d be left with a mix of non-durable frags which eventually would get reclaimed.\n\nThe 409 doesn\u0027t help much for replicated policies.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"2ede8d90_2f6986e6","in_reply_to":"f2dbba69_2dd79dc9","updated":"2025-09-25 06:14:18.000000000","message":"True, this patch only catches collisions on the same storage node and doesn\u0027t prevent all collisions, here is another extreme case: if fragments land on different nodes, collision might not be detected.\n\nAlso agree that this simple fix will likely resolve many of the issues we’ve seen. However, for a long-term solution, we’ll need to explore options such as ensuring timestamps are always unique.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d93eea271079c69c4cb1da441d2bc4540d9a344f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"4489bba7_9dffe624","updated":"2025-09-25 14:37:41.000000000","message":"+1 the change helps, but isn\u0027t a cure-all. The patch will need more work before merging.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"c76b8578_449e0e76","updated":"2025-12-09 18:20:32.000000000","message":"I still think this is useful","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"}],"swift/common/swob.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"10218b5002042d4d30f950c838eb59cab90ac621","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        else:"},{"line_number":1042,"context_line":"            # XXX remove this"},{"line_number":1043,"context_line":"            import socket"},{"line_number":1044,"context_line":"            if socket.gethostname() \u003d\u003d \u0027saio\u0027:"},{"line_number":1045,"context_line":"                ts \u003d Timestamp(int(time.time()))"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                ts \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":1,"id":"e4849af2_29e8125d","line":1044,"updated":"2024-08-28 19:13:36.000000000","message":"Would it be worth pulling this `socket.gethostname()` up to the module level so we do the `uname` syscall just once during process startup instead of on every client request?","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a77ef01566adbe078560c2d7287869e4c018a855","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        else:"},{"line_number":1042,"context_line":"            # XXX remove this"},{"line_number":1043,"context_line":"            import socket"},{"line_number":1044,"context_line":"            if socket.gethostname() \u003d\u003d \u0027saio\u0027:"},{"line_number":1045,"context_line":"                ts \u003d Timestamp(int(time.time()))"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                ts \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f89fe03_d9b159d7","line":1044,"in_reply_to":"e4849af2_29e8125d","updated":"2024-08-28 20:07:25.000000000","message":"I was thinking we could make it an explicit development configuration.  Do you remember vm_test_mode?\n\n468099: Remove deprecated vm_test_mode option | https://review.opendev.org/c/openstack/swift/+/468099\n\nMaybe \"testing_only_timestamp_collision_rounding \u003d False\"","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"}],"swift/common/utils/__init__.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":871,"context_line":"    \"\"\""},{"line_number":872,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":873,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":874,"context_line":"    overwrite target_path if it exists, we unlink and try again."},{"line_number":875,"context_line":""},{"line_number":876,"context_line":"    Attempts to fix / hide race conditions like empty object directories"},{"line_number":877,"context_line":"    being removed by backend processes during uploads, by retrying."}],"source_content_type":"text/x-python","patch_set":9,"id":"9b7f7e3a_42b4d95e","side":"PARENT","line":874,"updated":"2025-09-25 06:14:18.000000000","message":"docstring needs to be updated.","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":871,"context_line":"    \"\"\""},{"line_number":872,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":873,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":874,"context_line":"    overwrite target_path if it exists, and will raise OSError with"},{"line_number":875,"context_line":"    errno.EEXIST."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"}],"source_content_type":"text/x-python","patch_set":10,"id":"1eb3e91e_5f16b1ec","line":874,"range":{"start_line":874,"start_character":40,"end_line":874,"end_character":48},"updated":"2025-10-01 21:53:27.000000000","message":"\"this will\"?","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":true,"context_lines":[{"line_number":872,"context_line":"    \"\"\""},{"line_number":873,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":874,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":875,"context_line":"    overwrite target_path if it exists, we unlink and try again."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"},{"line_number":878,"context_line":"    being removed by backend processes during uploads, by retrying."}],"source_content_type":"text/x-python","patch_set":12,"id":"7b32d732_49c418fb","line":875,"updated":"2025-12-09 18:20:32.000000000","message":"I quite liked that the potential for OSError was mentioned here\n\nHow about:\n\n```\nlinkat() cannot overwrite target_path if it exists, so this function will retry but may ultimately raise OSError.\n```","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":true,"context_lines":[{"line_number":883,"context_line":"                         be fsync\u0027d."},{"line_number":884,"context_line":"    :param retries: number of retries to make"},{"line_number":885,"context_line":"    :param fsync: fsync on containing directory of target_path and also all"},{"line_number":886,"context_line":"                  the newly created directories."},{"line_number":887,"context_line":"    \"\"\""},{"line_number":888,"context_line":"    dirpath \u003d os.path.dirname(target_path)"},{"line_number":889,"context_line":"    attempts \u003d 0"}],"source_content_type":"text/x-python","patch_set":12,"id":"c46c443a_1cb3f788","line":886,"updated":"2025-12-09 18:20:32.000000000","message":"let\u0027s add\n\n```\n:raises: IOError if linkat fails more than ``retries`` times.\n```","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"}],"test/probe/test_timestamp_collision.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":39,"context_line":"        self.waiting \u003d None"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def track(self, req):"},{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"b5f23463_de645284","line":42,"updated":"2025-02-05 16:09:01.000000000","message":"it\u0027s a little hard to see; this is mapping the id of the request to an integer:\n\n0 is the first request\n1 is the second request\n\nThe zipper only ever supports two requests.\n\nthe setdefault returns (0, 1) and expects the caller to send that integer back in with each call to wait.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":39,"context_line":"        self.waiting \u003d None"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def track(self, req):"},{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"28f52f31_a0c3c015","line":42,"in_reply_to":"b5f23463_de645284","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"},{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"}],"source_content_type":"text/x-python","patch_set":3,"id":"d31c40a9_f447efb5","line":45,"updated":"2025-02-05 16:09:01.000000000","message":"req_id here is our tracking id; either (0, 1) so this is just calculating the \"other_id\"","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"},{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"}],"source_content_type":"text/x-python","patch_set":3,"id":"54e1c9e4_81c89a1a","line":45,"in_reply_to":"d31c40a9_f447efb5","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"},{"line_number":49,"context_line":"            self.win[my_id] \u003d fi"},{"line_number":50,"context_line":"            self.loose[other_id] \u003d fi"},{"line_number":51,"context_line":"        else:"},{"line_number":52,"context_line":"            self.loose[my_id] \u003d fi"}],"source_content_type":"text/x-python","patch_set":3,"id":"ee3c9c8f_89cbf674","line":49,"updated":"2025-02-05 16:09:01.000000000","message":"this can\u0027t be what was intended; `self.win` was originally a list of two sets?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"},{"line_number":49,"context_line":"            self.win[my_id] \u003d fi"},{"line_number":50,"context_line":"            self.loose[other_id] \u003d fi"},{"line_number":51,"context_line":"        else:"},{"line_number":52,"context_line":"            self.loose[my_id] \u003d fi"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f6006b1_b9ef3450","line":49,"in_reply_to":"ee3c9c8f_89cbf674","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":195,"context_line":"        )[:-300000]  # last frag is empty, second to last is short"},{"line_number":196,"context_line":"        now \u003d Timestamp.now()"},{"line_number":197,"context_line":"        orig_headers \u003d {"},{"line_number":198,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":199,"context_line":"        }"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        original_transfer_data \u003d ECObjectController._transfer_data"}],"source_content_type":"text/x-python","patch_set":3,"id":"f1425e4c_039aca96","line":198,"updated":"2025-02-05 16:09:01.000000000","message":"\"merely\" doing two uploads with the same timestamp one after the other will consistently result in only a single encrypted-etag on disk.\n\nIf the first one finishes the second write will get 409s or maybe go to handoffs.\n\nIf the first one is still going when the second one starts; by time the second one finishes it will unconditionally call rename and blat out all of the first ones frags and overwrite *all* the encrypted etags from the first.\n\nThe only way \"two simultaneous writes\" can result in a mix of encrypted-etags is if after they manage to get a timestamp collision they also have their individual object server requests from each also finish inconsistently.  It\u0027s a fascinating distributed race that you only get to see at massive scale.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6dffbc7e7b97f29b3e90a8e06ff588d3626bdd21","unresolved":true,"context_lines":[{"line_number":195,"context_line":"        )[:-300000]  # last frag is empty, second to last is short"},{"line_number":196,"context_line":"        now \u003d Timestamp.now()"},{"line_number":197,"context_line":"        orig_headers \u003d {"},{"line_number":198,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":199,"context_line":"        }"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        original_transfer_data \u003d ECObjectController._transfer_data"}],"source_content_type":"text/x-python","patch_set":3,"id":"d75a204c_fb0ebd31","line":198,"in_reply_to":"f1425e4c_039aca96","updated":"2025-08-29 04:18:21.000000000","message":"Can we take a peice from the lamport clock algorithm and just make every proxy send a unique x-timestamp? Fact is, we already have the ability to add a counter to the timestamp, that\u0027s what the offset of a Timestamp object it. That is each proxy adds it\u0027s own unique offset to the timestamp.. then they\u0027ll never collide. But otherwise objects are always treated as ts.normal which is the right timestamp?\n\nBetter, use something like a unique offset + pid so each worker is unique.. going further if eventlet threads are also a case throw in some random number into the request env as it passes through our swifthttp class that can be used to unique out originating requests\n\n   x-timestmap \u003d Timestamp(\u003cts\u003e, offset\u003d\u003cuniq_id\u003e + \u003cpid\u003e + \u003creq number\u003e)\n\nA unique id could just be generated from the mac address or something, cause it isn\u0027t something I\u0027d like to define manually.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":207,"context_line":"            # annotate the putters with tracking_id based off req"},{"line_number":208,"context_line":"            tracking_id \u003d zipper.track(req)"},{"line_number":209,"context_line":"            for putter in putters:"},{"line_number":210,"context_line":"                putter.__tracking_id \u003d tracking_id"},{"line_number":211,"context_line":"            return original_transfer_data("},{"line_number":212,"context_line":"                controller, req, policy, data_source, putters, nodes,"},{"line_number":213,"context_line":"                min_conns, etag_hasher)"}],"source_content_type":"text/x-python","patch_set":3,"id":"517254a0_2d48abfd","line":210,"updated":"2025-02-05 16:09:01.000000000","message":"if the putter already has a req instance it doesn\u0027t really need to hold on to the tracking_id on behalf of the zipper?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            # annotate the putters with tracking_id based off req"},{"line_number":208,"context_line":"            tracking_id \u003d zipper.track(req)"},{"line_number":209,"context_line":"            for putter in putters:"},{"line_number":210,"context_line":"                putter.__tracking_id \u003d tracking_id"},{"line_number":211,"context_line":"            return original_transfer_data("},{"line_number":212,"context_line":"                controller, req, policy, data_source, putters, nodes,"},{"line_number":213,"context_line":"                min_conns, etag_hasher)"}],"source_content_type":"text/x-python","patch_set":3,"id":"03683b89_3933b07d","line":210,"in_reply_to":"517254a0_2d48abfd","updated":"2025-09-05 22:01:06.000000000","message":"Done","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":218,"context_line":"            fi \u003d footer_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027]"},{"line_number":219,"context_line":"            orig_end_of_object_data(putter, footer_metadata)"},{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"4eb83913_4d3493c7","line":221,"updated":"2025-02-05 16:09:01.000000000","message":"I can imagine the tracking_id is mabye more helpful than id(req) - but we could print zipper.tracking_id[putter.req]","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":218,"context_line":"            fi \u003d footer_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027]"},{"line_number":219,"context_line":"            orig_end_of_object_data(putter, footer_metadata)"},{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"27a782ec_264be01f","line":221,"in_reply_to":"4eb83913_4d3493c7","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"}],"source_content_type":"text/x-python","patch_set":3,"id":"b9f3fa5e_a0490bae","line":223,"updated":"2025-02-05 16:09:01.000000000","message":"the requirement of this interface to pass back the internal tracking_id seems weird.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"}],"source_content_type":"text/x-python","patch_set":3,"id":"77316f16_189f9533","line":223,"in_reply_to":"b9f3fa5e_a0490bae","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"},{"line_number":227,"context_line":"                mock.patch.object(MIMEPutter, \u0027end_of_object_data\u0027,"},{"line_number":228,"context_line":"                                  patched_end_of_object_data):"},{"line_number":229,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":230,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":231,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"0c54728d_8940978b","line":228,"updated":"2025-02-05 16:09:01.000000000","message":"this patching isn\u0027t really changing what _transfer_data or end_of_object_data DO - just giving the test a hook to call `wait` in order to force the two `do_upload` streams to interleave such that we get the overlapping frags.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"},{"line_number":227,"context_line":"                mock.patch.object(MIMEPutter, \u0027end_of_object_data\u0027,"},{"line_number":228,"context_line":"                                  patched_end_of_object_data):"},{"line_number":229,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":230,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":231,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"69dc21ae_8ff8558f","line":228,"in_reply_to":"0c54728d_8940978b","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"33faec0e_256f1487","line":248,"updated":"2025-02-05 16:09:01.000000000","message":"I think I\u0027d suppose the assertion might want to see is like how many frags have which different encrypted-etag?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6983434069772330db7c088366042a374f14e386","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"75cd9fcc_37a3df96","line":248,"in_reply_to":"33faec0e_256f1487","updated":"2025-08-12 22:42:48.000000000","message":"hey!  this tests still works to create overlapping frags!\n\n\n```\nvagrant@saio:~$ python3 /vagrant/.scratch/nv-swift-gizmos/bin/rebuild_frags.py 1755038107.63809.tar.gz \n{\n  \"1755038107.63809\": {\n    \"8e54a2b23489f9a635f5bbd278db659e\": {\n      \"count\": 2,\n      \"frags\": \"0,2\"\n    },\n    \"50d0313d43a3f6c65275a98d67f694d0\": {\n      \"count\": 4,\n      \"frags\": \"1,3,4,5\"\n    }\n  }\n}\nWARNING: discarding metadata ...\n{\n  \"X-Object-Meta-Foo\": \"bar\",\n  \"X-Object-Meta-Xtime\": \"t1\"\n}\nusing path: \u0027/AUTH_test/badtest/badnews\u0027\nusing metadata ... \n{\n  \"X-Object-Meta-Bar\": \"baz\",\n  \"X-Object-Meta-Xtime\": \"t2\"\n}\n0 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 262144\n1 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 524288\n2 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 786432\n3 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 1048576\n0 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 1310720\n1 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 1572864\n2 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 1835008\n3 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 2097152\n0 8e54a2b23489f9a635f5bbd278db659e provided 187144 / 2284296\n1 50d0313d43a3f6c65275a98d67f694d0 provided 187144 / 2471440\n2 8e54a2b23489f9a635f5bbd278db659e provided 187144 / 2658584\n3 50d0313d43a3f6c65275a98d67f694d0 provided 187144 / 2845728\nexpected 4c38d6ba3d586810e10603df2047a847 \u003d\u003d 4c38d6ba3d586810e10603df2047a847 2845728\n```\n\n... what\u0027s crazy is this is actually a reasonable approximation of what we observe in prod.  Except both uploads have the same user metadata too - just different crypto meta.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5830842fc254dfd922e52778fae5bd7d457d7aec","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfd8dbdb_ff5694ec","line":248,"in_reply_to":"75cd9fcc_37a3df96","updated":"2025-09-09 02:56:38.000000000","message":"yes, when I turned on encryption within my vasio, I saw proxy-server running into \"ETag mismatch\":\n```\nSep  8 21:53:45 saio proxy-server: Problem with fragment response: ETag mismatch (txn: tx05ab7dff95254a8287878-0068bf5069) (client_ip: 127.0.0.1)\n```\n\nand if I read the object in the end of probe test, ``test_overlap_data_write_streams`` failed because request returned 503.\n```\nFAILED swift/test/probe/test_timestamp_collision.py::TestECCollision::test_overlap_data_write_streams - swift.common.internal_client.UnexpectedResponse: Unexpected response: 503 Service Unavailable (b\u0027\u003chtml\u003e\u003ch...\n```","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"165836fd_25f6e2be","line":248,"in_reply_to":"bfd8dbdb_ff5694ec","updated":"2025-09-19 23:19:58.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":270,"context_line":"        # re-upload with same timestamp (add some metadata)"},{"line_number":271,"context_line":"        headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"}],"source_content_type":"text/x-python","patch_set":3,"id":"6235cb1a_337c1930","line":273,"updated":"2025-02-05 16:09:01.000000000","message":"i\u0027m not sure what this test is trying to prove; obviously the \"overlap\" is going to write to handoffs if the primary disk is down?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":277,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":278,"context_line":"        # but the save_file metadata presists!"}],"source_content_type":"text/x-python","patch_set":3,"id":"229d19b4_5c0a9878","line":275,"updated":"2025-02-05 16:09:01.000000000","message":"right, cause we used internal client to force the timestamp collision, and the encrypted etag doesn\u0027t effect the df path/name.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":279,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":280,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"f1b7777b_06b25af6","line":282,"updated":"2025-02-05 16:09:01.000000000","message":"these are such weird/specific assertions; maybe better as a `read_meta_from_files` helper and then you can:\n\n```\nnew_files \u003d []\norig_meta \u003d None\nfor f in df2node:\n    if f \u003d\u003d save_file:\n        orig_meta \u003d self.read_meta_from_files([f])\n    else:\n        new_files.append(f)\nself.assertEqual(1, len(orig_meta))\nself.assertNotIn(\u0027foo\u0027, orig_meta[0])\nnew_meta \u003d self.read_meta_from_files(new_files)\nself.assertFalse(any(\u0027foo\u0027 in meta for meta in new_meta), \u0027found foo in %r\u0027 % new_meta)\n```","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":279,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":280,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"6c9fa4f4_bf7dc444","line":282,"in_reply_to":"f1b7777b_06b25af6","updated":"2025-09-19 23:19:58.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("},{"line_number":286,"context_line":"            self.account, self.container_name, self.object_name)"},{"line_number":287,"context_line":"        self.assertEqual(contents, b\u0027\u0027.join(body_iter))"}],"source_content_type":"text/x-python","patch_set":3,"id":"c9984fee_062a7edb","line":284,"updated":"2025-02-05 16:09:01.000000000","message":"it might be more interesting if we saved nparity+1 of the orig files - then ran the reconstructor to swallow the handoffs and then download probably wouldn\u0027t work?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2010-2024 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":8,"id":"417b9529_9ffc2594","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":25},"updated":"2025-09-19 14:20:21.000000000","message":"it\u0027s 2025 ;-)","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2010-2024 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":8,"id":"eaf19e64_cdf43cee","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":25},"in_reply_to":"417b9529_9ffc2594","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":51,"context_line":"        # Two sets tracking which fragment indexes each request has sent."},{"line_number":52,"context_line":"        self.sent \u003d [set(), set()]"},{"line_number":53,"context_line":"        # Two sets tracking fragments where this request \"lost\""},{"line_number":54,"context_line":"        self.loose \u003d [set(), set()]"},{"line_number":55,"context_line":"        # Two sets tracking fragments where this request \"won\""},{"line_number":56,"context_line":"        self.win \u003d [set(), set()]"},{"line_number":57,"context_line":"        self.waiting \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"28f876b7_b052ead8","line":54,"range":{"start_line":54,"start_character":13,"end_line":54,"end_character":18},"updated":"2025-09-19 14:20:21.000000000","message":"s/loose/lose/\n\n\"loose\" has a different meaning","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":51,"context_line":"        # Two sets tracking which fragment indexes each request has sent."},{"line_number":52,"context_line":"        self.sent \u003d [set(), set()]"},{"line_number":53,"context_line":"        # Two sets tracking fragments where this request \"lost\""},{"line_number":54,"context_line":"        self.loose \u003d [set(), set()]"},{"line_number":55,"context_line":"        # Two sets tracking fragments where this request \"won\""},{"line_number":56,"context_line":"        self.win \u003d [set(), set()]"},{"line_number":57,"context_line":"        self.waiting \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"25076a4b_cc892074","line":54,"range":{"start_line":54,"start_character":13,"end_line":54,"end_character":18},"in_reply_to":"28f876b7_b052ead8","updated":"2025-09-25 06:14:18.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":76,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":79,"context_line":"            # request that sent this fragment wins this fragment"},{"line_number":80,"context_line":"            self.win[my_id].add(fi)"},{"line_number":81,"context_line":"            self.loose[other_id].add(fi)"},{"line_number":82,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"66917782_86f9c187","line":79,"updated":"2025-09-19 14:20:21.000000000","message":"``if fi in self.sent[other_id]`` implies the other request has already sent this frag, so why does this clause then proceed to record *this* request as the winner and the other as the loser?\n\nIt probably makes no difference to the logic but it seems that the naming has been inverted such that self.win is recording the losing frags??","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":true,"context_lines":[{"line_number":76,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":79,"context_line":"            # request that sent this fragment wins this fragment"},{"line_number":80,"context_line":"            self.win[my_id].add(fi)"},{"line_number":81,"context_line":"            self.loose[other_id].add(fi)"},{"line_number":82,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"0f74781a_e8f5e53c","line":79,"in_reply_to":"66917782_86f9c187","updated":"2025-09-19 21:27:59.000000000","message":"on master the behavior was \"last write wins\" - I think we the error on link_at we invert the \"winner\"","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"        # check if request should wait"},{"line_number":87,"context_line":"        if (len(self.win[my_id]) \u003e len(self.win[other_id]) or"},{"line_number":88,"context_line":"                len(self.loose[my_id]) \u003e len(self.loose[other_id])):"},{"line_number":89,"context_line":"            if not self.waiting:  # prevent both requests from waiting"},{"line_number":90,"context_line":"                e \u003d self.waiting \u003d event.Event()"},{"line_number":91,"context_line":"                print(my_id, \u0027waiting\u0027, self.sent)"}],"source_content_type":"text/x-python","patch_set":8,"id":"62e80766_71d2dad6","line":88,"range":{"start_line":88,"start_character":16,"end_line":88,"end_character":66},"updated":"2025-09-19 14:20:21.000000000","message":"I\u0027m failing to understand this condition: if my_id has lost more times than   other_id, why does it now wait and presumably lose again?\n\nI also don\u0027t really understand the significance of win vs lose: isn\u0027t is sufficient for one thread to wait if it has sent more frags than the other?\n\nI changes this condition to \n\n```\nif len(self.sent[my_id]) \u003e len(self.sent[other_id]):\n```\n and the test still passes (I ran it \u003e\u003e10 times).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":8,"id":"03d27542_8ca75767","line":111,"range":{"start_line":111,"start_character":61,"end_line":111,"end_character":62},"updated":"2025-09-19 14:20:21.000000000","message":"is there a reason to want 3 attempts per request? I don\u0027t think we want the test to hide failed requests - see my comment at line 195","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":8,"id":"abb2e9f2_65553416","line":111,"range":{"start_line":111,"start_character":61,"end_line":111,"end_character":62},"in_reply_to":"03d27542_8ca75767","updated":"2025-09-19 21:27:59.000000000","message":"yeah, no that was wrong.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                    file_2_key[f].append(key)"},{"line_number":155,"context_line":"        return file_2_key"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def assertDatafilesMissingMetaKey(self, data_files, key):"},{"line_number":158,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":159,"context_line":"        errs \u003d defaultdict(list)"},{"line_number":160,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"08e6d0de_57384534","line":157,"updated":"2025-09-19 14:20:21.000000000","message":"these assertion helpers will pass if ``data_files`` is empty","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":154,"context_line":"                    file_2_key[f].append(key)"},{"line_number":155,"context_line":"        return file_2_key"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def assertDatafilesMissingMetaKey(self, data_files, key):"},{"line_number":158,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":159,"context_line":"        errs \u003d defaultdict(list)"},{"line_number":160,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3a891c8f_cc831496","line":157,"in_reply_to":"08e6d0de_57384534","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"4f51a2bd_065f1364","line":168,"updated":"2025-09-19 14:20:21.000000000","message":"nit (emphasize nit!): maybe follow the unittest naming pattern for containment assertions e.g. ``assertInDatafileMeta``, ``assertNotInDatafileMeta``\n\nor, another suggestion, just use a helper to gather the metadata dicts:\n\n```\nlist_of_dicts \u003d gather_datafile_metadata(data_files)\nassertTrue(all(key in d for d in list_of_dicts))\n```\n\nsee also line 295","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"fa00459c_e6498718","line":168,"in_reply_to":"1deb0afd_01ca1db2","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1deb0afd_01ca1db2","line":168,"in_reply_to":"4f51a2bd_065f1364","updated":"2025-09-19 21:27:59.000000000","message":"haha, yeah I think there\u0027s probably a better way to do this!","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"        orig_send_commit_confirmation \u003d MIMEPutter.send_commit_confirmation"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"}],"source_content_type":"text/x-python","patch_set":8,"id":"44df7670_79d1093f","line":192,"updated":"2025-09-19 14:20:21.000000000","message":"Nice. I\u0027m encouraged to see this mocking of internal client working to test concurrency because I think I\u0027ll need to do this kind of thing for native mpu 😊 Perhaps we already do it elsewhere and I\u0027m just not familiar.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"        orig_send_commit_confirmation \u003d MIMEPutter.send_commit_confirmation"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"}],"source_content_type":"text/x-python","patch_set":8,"id":"7a1a199a_0469442c","line":192,"in_reply_to":"44df7670_79d1093f","updated":"2025-09-19 21:27:59.000000000","message":"yeah I don\u0027t know where all we do this - i\u0027ve done it before to a pached internal-client to make POST requests to data files in the \"wrong\" storage policy index.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"},{"line_number":196,"context_line":"            return orig_send_commit_confirmation(putter)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"        with mock.patch.object(MIMEPutter, \u0027send_commit_confirmation\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"816cbb89_1b3e61a6","line":195,"updated":"2025-09-19 14:20:21.000000000","message":"I was playing with the test and wanted to check that the test would fail if the first request (the one that waits for proceed) failed. So I raised an exception right here, and the test still passed! I figured out that was because the internal client retried the request and the retry does not enter this if clause. But I think I\u0027d prefer the request to be asserting that the *first attempt* to commit the paused racing request *does succeed*, so I\u0027d suggest the the InternalClient should be configured to make only one attempt (and with that change my exception caused the test to fail as expected).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"},{"line_number":196,"context_line":"            return orig_send_commit_confirmation(putter)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"        with mock.patch.object(MIMEPutter, \u0027send_commit_confirmation\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"16ace967_ad8f1b68","line":195,"in_reply_to":"816cbb89_1b3e61a6","updated":"2025-09-19 21:27:59.000000000","message":"yeah i agree - retries is the default and not helpful at all in this patched test context.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":202,"context_line":"            gt \u003d spawn(self.do_upload, contents, headers)"},{"line_number":203,"context_line":"            # let the first thread run up to commit"},{"line_number":204,"context_line":"            ready.wait()"},{"line_number":205,"context_line":"            # we should have a bunch of non-durable"},{"line_number":206,"context_line":"            orig_df2node \u003d self.map_data_file_to_node()"},{"line_number":207,"context_line":"            self.assertDatafilesHaveMetaKey(orig_df2node, \u0027foo\u0027)"},{"line_number":208,"context_line":"            # overwrite from another thread"}],"source_content_type":"text/x-python","patch_set":8,"id":"3d205d04_8f486f8a","line":205,"range":{"start_line":205,"start_character":31,"end_line":205,"end_character":51},"updated":"2025-09-19 14:20:21.000000000","message":"that should be asserted - I don\u0027t think there\u0027s any assertion that verifies there are *any* files","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":202,"context_line":"            gt \u003d spawn(self.do_upload, contents, headers)"},{"line_number":203,"context_line":"            # let the first thread run up to commit"},{"line_number":204,"context_line":"            ready.wait()"},{"line_number":205,"context_line":"            # we should have a bunch of non-durable"},{"line_number":206,"context_line":"            orig_df2node \u003d self.map_data_file_to_node()"},{"line_number":207,"context_line":"            self.assertDatafilesHaveMetaKey(orig_df2node, \u0027foo\u0027)"},{"line_number":208,"context_line":"            # overwrite from another thread"}],"source_content_type":"text/x-python","patch_set":8,"id":"4917c460_b2b77611","line":205,"range":{"start_line":205,"start_character":31,"end_line":205,"end_character":51},"in_reply_to":"3d205d04_8f486f8a","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":209,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":210,"context_line":"            headers[\u0027x-object-meta-bar\u0027] \u003d \u0027baz\u0027"},{"line_number":211,"context_line":"            self.do_upload(contents, headers)"},{"line_number":212,"context_line":"            # they should all be durable"},{"line_number":213,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":214,"context_line":"            self.assertDatafilesMissingMetaKey(df2node, \u0027foo\u0027)"},{"line_number":215,"context_line":"            self.assertDatafilesHaveMetaKey(df2node, \u0027bar\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"28474cff_d9bb4123","line":212,"updated":"2025-09-19 14:20:21.000000000","message":"ditto, it would be good to actually assert the on disk files are as expected","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":209,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":210,"context_line":"            headers[\u0027x-object-meta-bar\u0027] \u003d \u0027baz\u0027"},{"line_number":211,"context_line":"            self.do_upload(contents, headers)"},{"line_number":212,"context_line":"            # they should all be durable"},{"line_number":213,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":214,"context_line":"            self.assertDatafilesMissingMetaKey(df2node, \u0027foo\u0027)"},{"line_number":215,"context_line":"            self.assertDatafilesHaveMetaKey(df2node, \u0027bar\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"951eb603_3a913bf4","line":212,"in_reply_to":"28474cff_d9bb4123","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":263,"context_line":"                                  patched_end_of_object_data):"},{"line_number":264,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":265,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":266,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"},{"line_number":267,"context_line":"            gt1 \u003d spawn(self.do_upload, contents, headers)"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"            headers \u003d dict(orig_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"79ca8f9a_e9f114e7","line":266,"updated":"2025-09-19 14:20:21.000000000","message":"AFAICT these xtime metadata are not asserted, and the use of ``t1`` and ``t2`` values is a little distracting given that the requests both have the same timestamp.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":263,"context_line":"                                  patched_end_of_object_data):"},{"line_number":264,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":265,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":266,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"},{"line_number":267,"context_line":"            gt1 \u003d spawn(self.do_upload, contents, headers)"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"            headers \u003d dict(orig_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"0ed840f6_c3122e25","line":266,"in_reply_to":"79ca8f9a_e9f114e7","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":278,"context_line":"            except Timeout:"},{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"}],"source_content_type":"text/x-python","patch_set":8,"id":"6bbfee55_9d04c31e","line":281,"updated":"2025-09-19 14:20:21.000000000","message":"it would be good to assert there are durable data files","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            except Timeout:"},{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"}],"source_content_type":"text/x-python","patch_set":8,"id":"32bbecdb_be52a332","line":281,"in_reply_to":"6bbfee55_9d04c31e","updated":"2025-09-19 21:27:59.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"}],"source_content_type":"text/x-python","patch_set":8,"id":"306a429a_7bbdc7ca","line":282,"updated":"2025-09-19 14:20:21.000000000","message":"the previous 2 lines cam be de-dented - I like to exit mock\u0027s ASAP","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"}],"source_content_type":"text/x-python","patch_set":8,"id":"0512e091_f70ae491","line":282,"in_reply_to":"306a429a_7bbdc7ca","updated":"2025-09-19 21:27:59.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"},{"line_number":286,"context_line":"        # Check that metadata for \u0027foo\u0027 exists, either encrypted or not"},{"line_number":287,"context_line":"        foo_keys \u003d self._meta_keys(\u0027foo\u0027)"},{"line_number":288,"context_line":"        self.assertTrue(any(key in all_found_keys for key in foo_keys),"}],"source_content_type":"text/x-python","patch_set":8,"id":"0106eb4f_18ca8610","line":285,"updated":"2025-09-19 14:20:21.000000000","message":"this just reduces to \n\n```\nall_keys \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027]).values()\n        all_found_keys \u003d [key for keys in all_keys for key in keys]\n```\n\nor make capture_metadata return the flattened list of all keys?","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"},{"line_number":286,"context_line":"        # Check that metadata for \u0027foo\u0027 exists, either encrypted or not"},{"line_number":287,"context_line":"        foo_keys \u003d self._meta_keys(\u0027foo\u0027)"},{"line_number":288,"context_line":"        self.assertTrue(any(key in all_found_keys for key in foo_keys),"}],"source_content_type":"text/x-python","patch_set":8,"id":"9faf0ed1_c2969fa6","line":285,"in_reply_to":"0106eb4f_18ca8610","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":292,"context_line":"        bar_keys \u003d self._meta_keys(\u0027bar\u0027)"},{"line_number":293,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":294,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":295,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":298,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":8,"id":"78464ffe_89cc5080","line":295,"updated":"2025-09-19 14:20:21.000000000","message":"this is similar to the assertDatafiles[Missing|Have]MetaKey, but uses a different pattern - it would be nice to unify the patterns, perhaps by having one helper to gather all the relevant metadata and then make assertions like this one (i.e. line 293, using ``any`` or ``all`` as appropriate).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":292,"context_line":"        bar_keys \u003d self._meta_keys(\u0027bar\u0027)"},{"line_number":293,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":294,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":295,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":298,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":8,"id":"ffa610eb_3e7b1d53","line":295,"in_reply_to":"78464ffe_89cc5080","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":322,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":323,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":324,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":325,"context_line":"        # but the save_file metadata presists!"},{"line_number":326,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":327,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":328,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"}],"source_content_type":"text/x-python","patch_set":8,"id":"92bc82c6_6daa1827","line":325,"range":{"start_line":325,"start_character":37,"end_line":325,"end_character":45},"updated":"2025-09-19 14:20:21.000000000","message":"1) typo: persists\n\n2) it\u0027s not just that the *metadata* persists, it\u0027s that the *data file* persists because the node it was on has been offline during the overwrite","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":322,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":323,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":324,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":325,"context_line":"        # but the save_file metadata presists!"},{"line_number":326,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":327,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":328,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"}],"source_content_type":"text/x-python","patch_set":8,"id":"95248093_7fca1926","line":325,"range":{"start_line":325,"start_character":37,"end_line":325,"end_character":45},"in_reply_to":"92bc82c6_6daa1827","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":true,"context_lines":[{"line_number":281,"context_line":""},{"line_number":282,"context_line":"        def safe_upload(contents, headers):"},{"line_number":283,"context_line":"            try:"},{"line_number":284,"context_line":"                resp \u003d self.do_upload(contents, headers)"},{"line_number":285,"context_line":"            except internal_client.UnexpectedResponse as e:"},{"line_number":286,"context_line":"                resp \u003d e.resp"},{"line_number":287,"context_line":"            results.append(resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"e24cd9d4_5034c4de","line":284,"updated":"2025-09-23 15:23:02.000000000","message":"self.do_upload doesn\u0027t return anything, nor does self.swift.upload_object 😞","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":true,"context_lines":[{"line_number":309,"context_line":"        df2node \u003d self.map_data_file_to_node()"},{"line_number":310,"context_line":"        self.assertEqual(self.policy.ec_n_unique_fragments, len(df2node))"},{"line_number":311,"context_line":"        # nothing is durable"},{"line_number":312,"context_line":"        self.assertEqual(0, len(self._collect_durable_files(df2node)))"},{"line_number":313,"context_line":"        # both responses were errors"},{"line_number":314,"context_line":"        self.assertEqual([503, 503], [r.status_int for r in results])"},{"line_number":315,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"4c98a012_a5b52303","line":312,"updated":"2025-09-23 15:23:02.000000000","message":"I reverted utils to master and as expected this fails; I made a dif to be totally sure the zipper was mixing up the metadata when the files were all durable:\n\n```\ndiff --git a/test/probe/test_timestamp_collision.py b/test/probe/test_timestamp_collision.py\nindex 6c32d290c..c8c0c40f8 100644\n--- a/test/probe/test_timestamp_collision.py\n+++ b/test/probe/test_timestamp_collision.py\n@@ -308,12 +308,7 @@ class TestECCollision(ECProbeTest):\n \n         df2node \u003d self.map_data_file_to_node()\n         self.assertEqual(self.policy.ec_n_unique_fragments, len(df2node))\n-        # nothing is durable\n-        self.assertEqual(0, len(self._collect_durable_files(df2node)))\n-        # both responses were errors\n-        self.assertEqual([503, 503], [r.status_int for r in results])\n \n-        # metadata is all mixed up!\n         counts \u003d defaultdict(int)\n         metadata \u003d self._collect_datafile_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])\n         foo_keys \u003d self._meta_keys(\u0027foo\u0027)\n@@ -326,6 +321,12 @@ class TestECCollision(ECProbeTest):\n                 self.assertTrue(any(key in m for key in bar_keys))\n                 counts[\u0027bar\u0027] +\u003d 1\n                 self.assertFalse(any(key in m for key in foo_keys))\n+\n+        # nothing is durable\n+        self.assertEqual(0, len(self._collect_durable_files(df2node)), counts)\n+        # both responses were errors\n+        self.assertEqual([503, 503], [r.status_int for r in results])\n+        # metadata is all mixed up!\n         # OMM I get all of these scenarios!?  I wonder if there\u0027s a way to make\n         # this more stable?\n         self.assertIn(counts, [\n\n```\n\nso I see\n\n```\n# nothing is durable\n\u003e       self.assertEqual(0, len(self._collect_durable_files(df2node)), counts)\nE       AssertionError: 0 !\u003d 6 : defaultdict(\u003cclass \u0027int\u0027\u003e, {\u0027foo\u0027: 3, \u0027bar\u0027: 3})\n```","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c7998f61_dff0dca5","line":377,"updated":"2025-09-19 23:19:58.000000000","message":"I think there\u0027s another df out on the hand off - might be interesting to throw some reconstructor into the mix here.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"}]}