)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"},{"line_number":29,"context_line":"you were on master today (excpet the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: Ib4690e274e3d05d4da3ad1322e1809b56f60288b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"95236b5a_ad30f5b1","line":29,"updated":"2025-09-25 06:14:18.000000000","message":"s/excpet/except","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d93eea271079c69c4cb1da441d2bc4540d9a344f","unresolved":true,"context_lines":[{"line_number":13,"context_line":"return a 503 response.  Well behaved clients should try and re-upload a"},{"line_number":14,"context_line":"PUT that returns 503."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"This patch is an improvement in that we won\u0027t return a 201 to a client"},{"line_number":17,"context_line":"unless a durable and consistent frag set is written."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"But we\u0027ll need more work to ensure that a set of non-durable frags are"},{"line_number":20,"context_line":"guaranteed to be or become consistent.  Even with this change, it should"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"4b6e2647_e9582fa2","line":17,"range":{"start_line":16,"start_character":37,"end_line":17,"end_character":52},"updated":"2025-09-25 14:37:41.000000000","message":"repeating myself: the additional probe tests in https://review.opendev.org/c/openstack/swift/+/962003/3 show this isn\u0027t true: we can still return 201 to both clients and have inconsistent but durable frag (although there will at least be a quorum of consistent frags).","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":13,"context_line":"return a 503 response.  Well behaved clients should try and re-upload a"},{"line_number":14,"context_line":"PUT that returns 503."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"This patch is an improvement in that we won\u0027t return a 201 to a client"},{"line_number":17,"context_line":"unless a durable and consistent frag set is written."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"But we\u0027ll need more work to ensure that a set of non-durable frags are"},{"line_number":20,"context_line":"guaranteed to be or become consistent.  Even with this change, it should"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"2f5919ef_c3aef35b","line":17,"range":{"start_line":16,"start_character":37,"end_line":17,"end_character":52},"in_reply_to":"4b6e2647_e9582fa2","updated":"2026-05-13 03:24:34.000000000","message":"ok, I feel like this commit message is basically bending over backwards trying to say it HASN\u0027T addressed timestamp collisions - and yet for 6mo I have NOT been busting out the frag repair tools; this isn\u0027t by chance - this was USEFUL.\n\nPlease don\u0027t feel like you\u0027re having to repeat yourself b/c you\u0027re not being heard; if you think this commit DOESN\u0027T say it DOESN\u0027T fix the bug HARD ENOUGH please offer alternative words you\u0027d find acceptable:\n\ns/durable and consistent/durable quorum of consistent/","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":22,"context_line":"durables with the same timestamp - but we don\u0027t know how to write that"},{"line_number":23,"context_line":"test yet [1]."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"3b8dc6c4_7d48916c","line":25,"updated":"2025-10-01 21:53:27.000000000","message":"Do we intend to do anything about it? Maybe start with writing a bug for it? Or is it more of a \"note\" than a \"todo\"?\n\nI guess maybe we could get the object-server to return 409? Then if one of the clients gets a 201, the other should get a 409 instead of a 503, right? I think that\u0027ll also address some of Alistair\u0027s concerns.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":22,"context_line":"durables with the same timestamp - but we don\u0027t know how to write that"},{"line_number":23,"context_line":"test yet [1]."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"79f7e29b_2acc66f6","line":25,"in_reply_to":"3b8dc6c4_7d48916c","updated":"2026-05-13 03:24:34.000000000","message":"I think what we \"intend TODO\" is v2-timestamp; this seems like it was and will still continue to be a useful improvement for the linkat utils helper (where the unlink and overwrite was relatively unexpected)\n\ns/TODO/NOTE/ in the commit message; I don\u0027t think there\u0027s an actionable follow-up lp bug that we can write.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"},{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"d53231e1_b16925a3","line":27,"range":{"start_line":27,"start_character":38,"end_line":27,"end_character":51},"updated":"2025-10-01 21:53:27.000000000","message":"Decidedly -- though I suppose fixing it is out of scope for the moment.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":true,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"TODO: The object-server traceback is not very subtle."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"[1] In this case we expect ssync will unfortunately make the"},{"line_number":28,"context_line":"non-conforming frags durable leaving you basically in the same state as"},{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"e05de46f_8f856ead","line":27,"range":{"start_line":27,"start_character":38,"end_line":27,"end_character":51},"in_reply_to":"d53231e1_b16925a3","updated":"2026-05-13 03:24:34.000000000","message":"actually I think we finally did write that test...\n\n962003: WIP sq: make FragZipper configurable | https://review.opendev.org/c/openstack/swift/+/962003\n\n^ nm, we never got that test working reliably and it may not appear as useful v2 timestamp post-jitter.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: Ib4690e274e3d05d4da3ad1322e1809b56f60288b"},{"line_number":33,"context_line":"Co-Authored-By: Jianjian Huo \u003cjhuo@nvidia.com\u003e"},{"line_number":34,"context_line":"Co-Authored-By: Alistair Coles \u003calistairncoles@gmail.com\u003e"},{"line_number":35,"context_line":"Signed-off-by: Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"2fa40a60_ecf6d69f","line":32,"updated":"2025-10-01 21:53:27.000000000","message":"Should we add a\n\n\u003e Partial-Bug: #1971686\n\n? And we still need to think about how to handle folks without `O_TMPFILE` support....","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":29,"context_line":"you were on master today (except the proxy is guaranteed to be able to"},{"line_number":30,"context_line":"read from the consistent frag set; at least until you loose disks)"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: Ib4690e274e3d05d4da3ad1322e1809b56f60288b"},{"line_number":33,"context_line":"Co-Authored-By: Jianjian Huo \u003cjhuo@nvidia.com\u003e"},{"line_number":34,"context_line":"Co-Authored-By: Alistair Coles \u003calistairncoles@gmail.com\u003e"},{"line_number":35,"context_line":"Signed-off-by: Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"442a2f60_8040cb79","line":32,"in_reply_to":"2fa40a60_ecf6d69f","updated":"2026-05-13 03:24:34.000000000","message":"\u003e Partial-Bug: #1971686\n\nfor *sure*\n\n\u003e how to handle folks without O_TMPFILE support\n\nI think basically the same plan on how we handle the last mile \"be or become consistent\" - we\u0027re going to use different timestamp/filenames to consistently (if arbitrarily) ensure the conflicting sets are orthogonal.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a77ef01566adbe078560c2d7287869e4c018a855","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"74838202_f35c6568","updated":"2024-08-28 20:07:25.000000000","message":"When I was testing this again yesterday I felt like I was having trouble creating the mis-matched etag fragments with encryption and concurrent uploads.  I want to write a probetest that can more reliably reproduce the issue - but I haven\u0027t thought of a strategy that seems worth commiting to yet.","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"4834dc0f_fe8c9890","updated":"2025-02-05 16:09:01.000000000","message":"I know there\u0027s an issue going on here; but I think these tests could be better.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6983434069772330db7c088366042a374f14e386","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"edc327e3_ef0dfe4b","updated":"2025-08-12 22:42:48.000000000","message":"I should take a stab at improving these tests.  Although they do seem to work to recreate the issue if anyone else wanted to dig into and help.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"fdf69e06_8012732a","updated":"2025-09-19 14:20:21.000000000","message":"I pushed some changes addressing some (but not all) of my comments here https://review.opendev.org/c/openstack/swift/+/961802\n\nI may need some help understanding the zipper class, but the tests seem to \"work\".","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"2993ec83_f2b6fca6","updated":"2025-09-19 21:27:59.000000000","message":"I think many of these comments will be addressed in 961802: sq: test_timestamp_collision fixups | https://review.opendev.org/c/openstack/swift/+/961802 - which we should squash!","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c9244a53_009c8d7d","updated":"2025-09-23 15:23:02.000000000","message":"I have a follow-up that makes the probe test more predictable, and adds another test that creates a partial but durable frag set. https://review.opendev.org/c/openstack/swift/+/962003","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ac4dd29ecf20ae11cab397b5f3b39526e0d4c8de","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"f2dbba69_2dd79dc9","updated":"2025-09-24 10:10:31.000000000","message":"I\u0027ve added a test in my follow-up patch ``test_overlap_data_write_streams_both_succeed_one_durable_set`` that shows a scenario where we can still have both requests return 201 to the client but have mixed durable frags on disk:\n\n* req2 writes fragX\n* req1 fails to write fragX\n* req1 writes N-1 other frags\n* req2 fails to write fragY\n* req1 got N-1 ok frags so makes N-1 frags durable\n* req1 -\u003e 201 to client\n* req2 writes N-2 other frags over the durable frags!! \n* req 2 got N-2 plus fragX ok frags, i.e. N-1, so makes them durable\n* req2 -\u003e 201 to client\n\nThat does NOT mean this patch is not necessary, but it does mean that this patch cannot claim to always return 503 on a client timestamp collision.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"8cabf269_6c7a686b","updated":"2025-09-19 23:19:58.000000000","message":"we\u0027ll see if zuul likes it and we can keep writing some more tests","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"bfc98405_28ad0531","in_reply_to":"b0ce7134_a3abf3e2","updated":"2026-05-13 03:24:34.000000000","message":"\u003e the proxy should NOT proceed to the commit phase if any frag returns 409.\n\nI think that would be scope creep; a 500 with a traceback was a *significant* improvement over \"both requests return 201\"\n\nWe could maybe discuss the possibility/need to drag in *just* the object-server change:\n\n```\ndiff --git a/swift/obj/server.py b/swift/obj/server.py\nindex cd4be9dbf1..6dd98e603f 100644\n--- a/swift/obj/server.py\n+++ b/swift/obj/server.py\n@@ -1084,7 +1084,10 @@ class ObjectController(BaseStorageServer):\n             else:\n                 footers_metadata \u003d {}\n             self._apply_extra_metadata(request, metadata, footers_metadata)\n-            writer.put(metadata)\n+            try:\n+                writer.put(metadata)\n+            except FileExistsError:\n+                return HTTPConflict(request\u003drequest)\n             if multi_stage_mime_state:\n                 self._send_multi_stage_continue_headers(\n                     request, **multi_stage_mime_state)\n```\n\n... but I think it\u0027s harder to justify that doesn\u0027t require additional proxy level unittesting (i.e. in addition to probe tests) in order to validate the proxy behavior explicitly.  We sort of know/trust that object servers can return 500s and roughly that the proxy server kind of \"must\" not do anything STUPID on ServerError; so that\u0027s levering the RESTful interface boundary.  I think unique/special 4XX errors on PUT would require more careful attention and could easily be layered on as a \"follow-up\" instead of \"necessary for this change to be complete and correct on it\u0027s own merrits\"\n\n```\n        with mock.patch.object(ReplicatedObjectController, \u0027_make_putter\u0027,\n                               _patch_putter), \\\n                mock.patch.object(ReplicatedObjectController,\n                                  \u0027_get_put_responses\u0027,\n                                  _patch_get_put_responses), \\\n                self.assertRaises(internal_client.UnexpectedResponse) as ctx:\n            self.swift.upload_object(BytesIO(contents), self.account,\n                                     self.container_name, self.object_name,\n                                     headers\u003dheaders)\n\u003e       self.assertEqual(ctx.exception.resp.status_int, 503)\nE       AssertionError: 409 !\u003d 503\n```\n\n988379: wip: return 409 on linkat collision | https://review.opendev.org/c/openstack/swift/+/988379","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ec0fe93878008e2ec57a842f04065431f2e77d2d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"b0ce7134_a3abf3e2","in_reply_to":"f2dbba69_2dd79dc9","updated":"2025-09-25 14:32:13.000000000","message":"\u003e we can still have both requests return 201 to the client but have mixed durable frags on disk\n\nthinking about this more, perhaps the obj server should return 409 (conflict) if it gets EEXIST while linking to the .data file. Then the proxy should NOT proceed to the commit phase if any frag returns 409. That would significantly further narrow the opportunity to reach a set of mixed-metadata durable frags. Typically both requests would get a 503 due to one or more backend 409s, and we\u0027d be left with a mix of non-durable frags which eventually would get reclaimed.\n\nThe 409 doesn\u0027t help much for replicated policies.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"2ede8d90_2f6986e6","in_reply_to":"f2dbba69_2dd79dc9","updated":"2025-09-25 06:14:18.000000000","message":"True, this patch only catches collisions on the same storage node and doesn\u0027t prevent all collisions, here is another extreme case: if fragments land on different nodes, collision might not be detected.\n\nAlso agree that this simple fix will likely resolve many of the issues we’ve seen. However, for a long-term solution, we’ll need to explore options such as ensuring timestamps are always unique.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d93eea271079c69c4cb1da441d2bc4540d9a344f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"4489bba7_9dffe624","updated":"2025-09-25 14:37:41.000000000","message":"+1 the change helps, but isn\u0027t a cure-all. The patch will need more work before merging.","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"c76b8578_449e0e76","updated":"2025-12-09 18:20:32.000000000","message":"I still think this is useful","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":15,"id":"5f122376_a6a3a9c0","updated":"2026-05-13 03:24:34.000000000","message":"I think this patch is in pretty good shpae!  it\u0027s been useful/helpful - it\u0027s WELL tested to explore the problem space and it\u0027s limits...\n\nI could really use some pointers on what we actually think needs to be changed for this to merge; I don\u0027t see a path towards us to \"stop carrying it\"?\n\n... unless do we think once we have v2 timestamps we WANT the linkat/unlink behavior *back*!?  I feel like it was masking this bug in bad way!?  This is *better*, right?\n\n\u003e the change is otherwise complete correct and undeniably makes Swift better (not perfect, better).","commit_id":"cf1537b6af35ec9277e3d254f39afd38ae578070"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3ccf124ad8cd07694a0dab0dc4e62ef3877a4f96","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"309829a2_d804e56d","updated":"2026-05-13 05:38:36.000000000","message":"Also happy for things to be a follow up.","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c22919ecfe50a3ba569f81a6b9d33d3eee87d1ee","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"dab0a425_fb097b14","updated":"2026-05-13 09:34:23.000000000","message":"I have no strong argument against narrowing the metadata in the logged exception to just the useful/relevant sysmeta - i think that would be an improvement; feel free to do that or put up a squash or I\u0027ll get around to it next chance I get!","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d586fe9bb58c53c0be81ef621db9890c6ce14049","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"d4dc5bbc_8b8b889d","updated":"2026-05-13 05:38:00.000000000","message":"I know we carry this, and not really a blocker, but there is a potentual information breach when there is a collision in the logs.\n\nAlso this only does it the metadata check for O_TMPFILE should be make it compatible when it\u0027s just a tmp_file rename?","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":18,"id":"dc594fe1_02842937","updated":"2026-05-15 15:20:14.000000000","message":"I like everything about this patch except the probe tests. That\u0027s partly because I have not yet had time to thoroughly review the probe tests, so cannot vote for them, but also because I am not sure what their destiny will be in the light of timestamp v2. I don\u0027t want to maintain tests that in the future just document how things used to be broken. Do we imagine they will be updated to demonstrate how things can still break if/when two identical v2 timestamps are chosen?\n\nWould this patch be good enough to merge without the probe tests? If only to expedite progress and continue review/work on the probe tests.\n\nRe nits etc: I have pushed some suggested tweaks here: https://review.opendev.org/c/openstack/swift/+/988781","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3e9135012f2e4215b19883d3055f56edf6f089ad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":18,"id":"36c9fd48_8cba1109","updated":"2026-05-13 22:02:41.000000000","message":"ok matt back to you!\n\n```\nMay 13 22:02:26 saio object-server-6020: ERROR __call__ error with PUT /sdb2/712/AUTH_test/container-fcb6d62a-fe12-4e47-aefa-f9d1008f78fe/object-e7ca924d-1018-4c2f-b944-eac32e155f8d : #012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/obj/diskfile.py\", line 1971, in _safe_metadata_linkat#012    link_fd_to_path(self._fd, target_path,#012  File \"/vagrant/swift/swift/common/utils/__init__.py\", line 892, in link_fd_to_path#012    linkat(linkat.AT_FDCWD, \"/proc/self/fd/%d\" % (fd),#012  File \"/vagrant/swift/swift/common/linkat.py\", line 78, in __call__#012    return self._c_linkat(olddirfd, oldpath, newdirfd, newpath, flags)#012  File \"/vagrant/swift/swift/common/linkat.py\", line 48, in errcheck#012    raise IOError(errno, \u0027linkat: %s\u0027 % os.strerror(errno))#012FileExistsError: [Errno 17] linkat: File exists#012#012During handling of the above exception, another exception occurred:#012#012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/obj/server.py\", line 1414, in __call__#012    res \u003d getattr(self, req.method)(req)#012  File \"/vagrant/swift/swift/common/base_storage_server.py\", line 71, in _timing_stats#012    resp \u003d func(ctrl, *args, **kwargs)#012  File \"/vagrant/swift/swift/common/base_storage_server.py\", line 44, in _timing_stats#012    resp \u003d func(#012  File \"/vagrant/swift/swift/obj/server.py\", line 1087, in PUT#012    writer.put(metadata)#012  File \"/vagrant/swift/swift/obj/diskfile.py\", line 3444, in put#012    super(ECDiskFileWriter, self)._put(metadata, cleanup, frag_index\u003dfi)#012  File \"/vagrant/swift/swift/obj/diskfile.py\", line 2078, in _put#012    tpool.execute(#012  File \"/usr/local/lib/python3.10/dist-packages/eventlet/tpool.py\", line 125, in execute#012    raise e.with_traceback(tb)#012  File \"/usr/local/lib/python3.10/dist-packages/eventlet/tpool.py\", line 82, in tworker#012    rv \u003d meth(*args, **kwargs)#012  File \"/vagrant/swift/swift/obj/diskfile.py\", line 2022, in _finalize_put#012    self._safe_metadata_linkat(target_path, metadata)#012  File \"/vagrant/swift/swift/obj/diskfile.py\", line 1996, in _safe_metadata_linkat#012    raise FileExistsError(#012FileExistsError: Conflicting pre-existing metadata: {\"ETag\": \"7e3d7534ca4a3b70c43348a36612c2bc\", \"X-Object-Sysmeta-Ec-Etag\": \"23c42e11237c24b5b4e01513916dab4a\", \"X-Timestamp\": \"1778709746.28936\"} \u003d\u003d {\"ETag\": \"7e3d7534ca4a3b70c43348a36612c2bc\", \"X-Object-Sysmeta-Ec-Etag\": \"23c42e11237c24b5b4e01513916dab4a\", \"X-Timestamp\": \"1778709746.28936\"} (incoming) (txn: tx825b232c7b6c4abd8da2b-006a04f4f2)\n```\n\n\nI am realizing that despite linux doing o_tmpfile for 10 years all mac peeps don\u0027t have an equivilent to linkat that will EEXIST when coming out of mkstemp stuff; instead the recommended way is to grab a *different* \"non-posix\" libc function:\n\n```\nimport ctypes\nimport os\n\n# Load the standard C library on macOS\nlibc \u003d ctypes.CDLL(None)\n\n# Define the macOS flag for swapping\nRENAME_SWAP \u003d 0x00000002\n\ndef atomic_swap_macos(path1: str, path2: str):\n    \"\"\"\n    Atomically exchanges path1 and path2 using macOS renamex_np.\n    Both paths must exist beforehand.\n    \"\"\"\n    # int renamex_np(const char *from, const char *to, unsigned int flags);\n    result \u003d libc.renamex_np(\n        path1.encode(\u0027utf-8\u0027), \n        path2.encode(\u0027utf-8\u0027), \n        ctypes.c_uint(RENAME_SWAP)\n    )\n    if result !\u003d 0:\n        errno \u003d ctypes.get_errno()\n        raise OSError(errno, os.strerror(errno))\n```\n\n^ or we could say \"y u no use prod os in dev test!?\" or make a slower rename-LBYL path that\u0027s just for testing on mac.  I\u0027m pretty comfortable with the `@requires_otmpfile` for now and leaving the \"make mac testing better\" as follow-up; but I may be unreasonably biased having been directly impacted by all the 201s resulting in data that can\u0027t be read until we started carrying this!","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"55a8aa57e8f52ba931573926189ebba8d934682f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"9b5b557c_e452e80e","updated":"2026-05-29 21:47:05.000000000","message":"we need to fix these flakey tests:\n\n```\n  -1, the new probe tests in 9f7411de7 are not reliable as written.\n\n  This parent adds test/probe/test_timestamp_collision.py; the top change\n  419ed7458 / 990738 does not add probe tests.\n\n  I reproduced the CI failure locally on vsaio with a 5-run loop of:\n\n    test/probe/test_timestamp_collision.py::TestReplicatedCollision::test_request_race\n\n  In the captured loop, runs 1-4 passed and run 5 failed:\n\n    expected [[201, 201, 500], [201, 500, 500]]\n    got      [[201, 201, 201], [500, 500, 500]]\n\n  That matches the CI failure mode. The test triggers the second upload from\n  putter.end_of_object_data, but that only controls when the proxy sends final\n  bytes to backend connections. It does not force a specific object-server publish\n  ordering. Sometimes one request wins all three backend races before the other\n  publishes anywhere, so [201, 201, 201] / [500, 500, 500] is a real race outcome.\n```\n\nrepro should be:\n\n```\npytest -q -s test/probe/test_timestamp_collision.py::TestReplicatedCollision::test_request_race\n```\n\n^ in a loop!\n\nWe might find a solution somewhere in here:\n\n988537: wip: this more reliable; is it better? | https://review.opendev.org/c/openstack/swift/+/988537\n\nI don\u0027t think that\u0027s a blocker on continuing to carry the patch; but we should fix it (and probably remove some of those probetests) before we merge.\n\nI tried to get my agent to squash these:\n\n988537: wip: this more reliable; is it better? | https://review.opendev.org/c/openstack/swift/+/988537\n988781: sq? fixups re link_fd_to_path | https://review.opendev.org/c/openstack/swift/+/988781\n\nI think that worked but is probably worth another eagle eye\n\nI did *not* squash:\n\n990738: sq? handle named-temp collision races | https://review.opendev.org/c/openstack/swift/+/990738\n\n... but I sort of see the appeal if Matt or some non-linkat-tmp-having-developers are REALLY interested in the idea.","commit_id":"999f7db86071f9047286d24bf47c7dcc4021ecbb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8a8e513e03b8f0acf35d3a024b1ad80cafd8348f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"2235ad15_3097dabb","updated":"2026-06-02 21:59:47.000000000","message":"I think we should drop some of the un-needed probe tests:\n\n991231: sq: defer extra timestamp collision scenarios | https://review.opendev.org/c/openstack/swift/+/991231\n\n... and fix the flakey one:\n\n991230: test/probe: split overlap timestamp collision cases | https://review.opendev.org/c/openstack/swift/+/991230","commit_id":"c189d64890ed12f193f7d29248a0315cafc7ab37"}],"swift/common/swob.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"10218b5002042d4d30f950c838eb59cab90ac621","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        else:"},{"line_number":1042,"context_line":"            # XXX remove this"},{"line_number":1043,"context_line":"            import socket"},{"line_number":1044,"context_line":"            if socket.gethostname() \u003d\u003d \u0027saio\u0027:"},{"line_number":1045,"context_line":"                ts \u003d Timestamp(int(time.time()))"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                ts \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":1,"id":"e4849af2_29e8125d","line":1044,"updated":"2024-08-28 19:13:36.000000000","message":"Would it be worth pulling this `socket.gethostname()` up to the module level so we do the `uname` syscall just once during process startup instead of on every client request?","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":1041,"context_line":"        else:"},{"line_number":1042,"context_line":"            # XXX remove this"},{"line_number":1043,"context_line":"            import socket"},{"line_number":1044,"context_line":"            if socket.gethostname() \u003d\u003d \u0027saio\u0027:"},{"line_number":1045,"context_line":"                ts \u003d Timestamp(int(time.time()))"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                ts \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":1,"id":"5bae9e75_55a20c2b","line":1044,"in_reply_to":"9f89fe03_d9b159d7","updated":"2026-05-13 03:24:34.000000000","message":"Done","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a77ef01566adbe078560c2d7287869e4c018a855","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        else:"},{"line_number":1042,"context_line":"            # XXX remove this"},{"line_number":1043,"context_line":"            import socket"},{"line_number":1044,"context_line":"            if socket.gethostname() \u003d\u003d \u0027saio\u0027:"},{"line_number":1045,"context_line":"                ts \u003d Timestamp(int(time.time()))"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                ts \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f89fe03_d9b159d7","line":1044,"in_reply_to":"e4849af2_29e8125d","updated":"2024-08-28 20:07:25.000000000","message":"I was thinking we could make it an explicit development configuration.  Do you remember vm_test_mode?\n\n468099: Remove deprecated vm_test_mode option | https://review.opendev.org/c/openstack/swift/+/468099\n\nMaybe \"testing_only_timestamp_collision_rounding \u003d False\"","commit_id":"95a77af9a72f3bea4db2e142354bd0feb243d213"}],"swift/common/utils/__init__.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":871,"context_line":"    \"\"\""},{"line_number":872,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":873,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":874,"context_line":"    overwrite target_path if it exists, we unlink and try again."},{"line_number":875,"context_line":""},{"line_number":876,"context_line":"    Attempts to fix / hide race conditions like empty object directories"},{"line_number":877,"context_line":"    being removed by backend processes during uploads, by retrying."}],"source_content_type":"text/x-python","patch_set":9,"id":"9b7f7e3a_42b4d95e","side":"PARENT","line":874,"updated":"2025-09-25 06:14:18.000000000","message":"docstring needs to be updated.","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e98150f38ff8db5c72240396ad64cfc424a6894","unresolved":true,"context_lines":[{"line_number":871,"context_line":"    \"\"\""},{"line_number":872,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":873,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":874,"context_line":"    overwrite target_path if it exists, and will raise OSError with"},{"line_number":875,"context_line":"    errno.EEXIST."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"}],"source_content_type":"text/x-python","patch_set":10,"id":"1eb3e91e_5f16b1ec","line":874,"range":{"start_line":874,"start_character":40,"end_line":874,"end_character":48},"updated":"2025-10-01 21:53:27.000000000","message":"\"this will\"?","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":871,"context_line":"    \"\"\""},{"line_number":872,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":873,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":874,"context_line":"    overwrite target_path if it exists, and will raise OSError with"},{"line_number":875,"context_line":"    errno.EEXIST."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"}],"source_content_type":"text/x-python","patch_set":10,"id":"97e8c449_c1fb102e","line":874,"range":{"start_line":874,"start_character":40,"end_line":874,"end_character":48},"in_reply_to":"1eb3e91e_5f16b1ec","updated":"2026-05-13 03:24:34.000000000","message":"Done","commit_id":"00e8873798438cb5d409f3774ead31cd1a1f2ea2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":true,"context_lines":[{"line_number":872,"context_line":"    \"\"\""},{"line_number":873,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":874,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":875,"context_line":"    overwrite target_path if it exists, we unlink and try again."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"},{"line_number":878,"context_line":"    being removed by backend processes during uploads, by retrying."}],"source_content_type":"text/x-python","patch_set":12,"id":"7b32d732_49c418fb","line":875,"updated":"2025-12-09 18:20:32.000000000","message":"I quite liked that the potential for OSError was mentioned here\n\nHow about:\n\n```\nlinkat() cannot overwrite target_path if it exists, so this function will retry but may ultimately raise OSError.\n```","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":872,"context_line":"    \"\"\""},{"line_number":873,"context_line":"    Creates a link to file descriptor at target_path specified. This method"},{"line_number":874,"context_line":"    does not close the fd for you. Unlike rename, as linkat() cannot"},{"line_number":875,"context_line":"    overwrite target_path if it exists, we unlink and try again."},{"line_number":876,"context_line":""},{"line_number":877,"context_line":"    Attempts to fix / hide race conditions like empty object directories"},{"line_number":878,"context_line":"    being removed by backend processes during uploads, by retrying."}],"source_content_type":"text/x-python","patch_set":12,"id":"bfd01cfd_4d63872e","line":875,"in_reply_to":"7b32d732_49c418fb","updated":"2026-05-13 03:24:34.000000000","message":"Done","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"727822dd2828afc28e9cb64240e9a2cb42caa943","unresolved":true,"context_lines":[{"line_number":883,"context_line":"                         be fsync\u0027d."},{"line_number":884,"context_line":"    :param retries: number of retries to make"},{"line_number":885,"context_line":"    :param fsync: fsync on containing directory of target_path and also all"},{"line_number":886,"context_line":"                  the newly created directories."},{"line_number":887,"context_line":"    \"\"\""},{"line_number":888,"context_line":"    dirpath \u003d os.path.dirname(target_path)"},{"line_number":889,"context_line":"    attempts \u003d 0"}],"source_content_type":"text/x-python","patch_set":12,"id":"c46c443a_1cb3f788","line":886,"updated":"2025-12-09 18:20:32.000000000","message":"let\u0027s add\n\n```\n:raises: IOError if linkat fails more than ``retries`` times.\n```","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":883,"context_line":"                         be fsync\u0027d."},{"line_number":884,"context_line":"    :param retries: number of retries to make"},{"line_number":885,"context_line":"    :param fsync: fsync on containing directory of target_path and also all"},{"line_number":886,"context_line":"                  the newly created directories."},{"line_number":887,"context_line":"    \"\"\""},{"line_number":888,"context_line":"    dirpath \u003d os.path.dirname(target_path)"},{"line_number":889,"context_line":"    attempts \u003d 0"}],"source_content_type":"text/x-python","patch_set":12,"id":"0b1f291c_1d6e2808","line":886,"in_reply_to":"c46c443a_1cb3f788","updated":"2026-05-13 03:24:34.000000000","message":"I mean; we only retry on the `makedirs_count` path - for EEXISTS I think we raise immeidately.","commit_id":"a164d3e6c83b1bac19a72bdbc44a8536617eb3a2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":879,"context_line":"    :param target_path: Path in filesystem where fd is to be linked"},{"line_number":880,"context_line":"    :param dirs_created: Number of newly created directories that needs to"},{"line_number":881,"context_line":"                         be fsync\u0027d."},{"line_number":882,"context_line":"    :param retries: number of retries to make"},{"line_number":883,"context_line":"    :param fsync: fsync on containing directory of target_path and also all"},{"line_number":884,"context_line":"                  the newly created directories."},{"line_number":885,"context_line":"    :raises: IOError if linkat fails w/ ENOENT ``retries`` times or other"}],"source_content_type":"text/x-python","patch_set":18,"id":"a8a89a3d_eb7aac46","line":882,"updated":"2026-05-15 15:20:14.000000000","message":"does retries arg make sense any more?\n\nAFAICT it was never passed in so retries always was 2. That now allows for two attempts at the ``makedirs_count`` (vs one in ``renamer``).\n\nThe docstring should at least be more specific about what is retried.\n\nSuggestion:\n\n```\ndiff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py\nindex df040df52..4a8318c61 100644\n--- a/swift/common/utils/__init__.py\n+++ b/swift/common/utils/__init__.py\n@@ -873,17 +873,18 @@ def link_fd_to_path(fd, target_path, dirs_created\u003d0, retries\u003d2, fsync\u003dTrue):\n     target_path if it exists.\n \n     Attempts to fix / hide race conditions like empty object directories\n-    being removed by backend processes during uploads, by retrying.\n+    being removed by backend processes during uploads by making directories in\n+    the ``target_path``.\n \n     :param fd: File descriptor to be linked\n     :param target_path: Path in filesystem where fd is to be linked\n     :param dirs_created: Number of newly created directories that needs to\n                          be fsync\u0027d.\n-    :param retries: number of retries to make\n+    :param retries: number of times to retry making directories.\n     :param fsync: fsync on containing directory of target_path and also all\n                   the newly created directories.\n-    :raises: IOError if linkat fails w/ ENOENT ``retries`` times or other\n-             errors like EEXIST\n+    :raises: IOError if linkat fails with ENOENT ``retries`` times or fails\n+        once with other errors like EEXIST.\n     \"\"\"\n     dirpath \u003d os.path.dirname(target_path)\n     attempts \u003d 0\n\n```\n\nsee https://review.opendev.org/c/openstack/swift/+/988781","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"}],"swift/obj/diskfile.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d586fe9bb58c53c0be81ef621db9890c6ce14049","unresolved":true,"context_lines":[{"line_number":1975,"context_line":"            if existing_metadata !\u003d metadata:"},{"line_number":1976,"context_line":"                raise FileExistsError("},{"line_number":1977,"context_line":"                    \u0027Conflicting pre-existing metadata: %r !\u003d %r (on disk)\u0027 % ("},{"line_number":1978,"context_line":"                        metadata, existing_metadata))"},{"line_number":1979,"context_line":""},{"line_number":1980,"context_line":"    def _finalize_put(self, metadata, target_path, cleanup,"},{"line_number":1981,"context_line":"                      logger_thread_locals):"}],"source_content_type":"text/x-python","patch_set":16,"id":"6a8cf77f_8410341d","line":1978,"updated":"2026-05-13 05:38:00.000000000","message":"codex makes a really good point here. There is a chance there that user meta can unintentailly spill into the logs:\n\n```\nModerate: the raised FileExistsError includes full metadata dicts at swift/obj/diskfile.py:1976.\n     Because object-server turns unhandled exceptions into traceback bodies at swift/obj/server.py:1419,\n     and proxy logs up to 1024 bytes of 5xx bodies at swift/proxy/server.py:743, a client-triggerable\n     collision can spill user metadata/sysmeta into logs. I’d keep the exception message terse\n```\n\nDo we need to sanitise the output or do we really need to include the metadata that doesn\u0027t match?","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c22919ecfe50a3ba569f81a6b9d33d3eee87d1ee","unresolved":true,"context_lines":[{"line_number":1975,"context_line":"            if existing_metadata !\u003d metadata:"},{"line_number":1976,"context_line":"                raise FileExistsError("},{"line_number":1977,"context_line":"                    \u0027Conflicting pre-existing metadata: %r !\u003d %r (on disk)\u0027 % ("},{"line_number":1978,"context_line":"                        metadata, existing_metadata))"},{"line_number":1979,"context_line":""},{"line_number":1980,"context_line":"    def _finalize_put(self, metadata, target_path, cleanup,"},{"line_number":1981,"context_line":"                      logger_thread_locals):"}],"source_content_type":"text/x-python","patch_set":16,"id":"e4b0f432_ab731bac","line":1978,"in_reply_to":"6a8cf77f_8410341d","updated":"2026-05-13 09:34:23.000000000","message":"in prod we encrypt all metadata; but the error is needlessly verbose - the only things we care about are timestamp and sysmeta-ec-etag","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3e9135012f2e4215b19883d3055f56edf6f089ad","unresolved":true,"context_lines":[{"line_number":1975,"context_line":"            if existing_metadata !\u003d metadata:"},{"line_number":1976,"context_line":"                raise FileExistsError("},{"line_number":1977,"context_line":"                    \u0027Conflicting pre-existing metadata: %r !\u003d %r (on disk)\u0027 % ("},{"line_number":1978,"context_line":"                        metadata, existing_metadata))"},{"line_number":1979,"context_line":""},{"line_number":1980,"context_line":"    def _finalize_put(self, metadata, target_path, cleanup,"},{"line_number":1981,"context_line":"                      logger_thread_locals):"}],"source_content_type":"text/x-python","patch_set":16,"id":"6a3fa428_ef56826d","line":1978,"in_reply_to":"e4b0f432_ab731bac","updated":"2026-05-13 22:02:41.000000000","message":"```\nFileExistsError: Conlficting pre-existing metadata: {\u0027X-Timestamp\u0027: \u00271778706413.00000\u0027, \u0027ETag\u0027: \u0027existing-data-etag\u0027, \u0027X-Object-Sysmeta-Ec-Etag\u0027: \u0027existing-ec-etag\u0027} !\u003d {\u0027X-Timestamp\u0027: \u00271778706413.00000\u0027, \u0027ETag\u0027: \u00270b4c12d7e0a73840c1c4f148fda3b037\u0027, \u0027X-Object-Sysmeta-Ec-Etag\u0027: \u0027incoming-ec-etag\u0027} (incoming)\nFileExistsError: Conlficting pre-existing metadata: {\u0027X-Timestamp\u0027: \u00271778706413.00000\u0027, \u0027ETag\u0027: \u00270b4c12d7e0a73840c1c4f148fda3b037\u0027, \u0027X-Object-Sysmeta-Ec-Etag\u0027: \u0027same-ec-etag\u0027} \u003d\u003d {\u0027X-Timestamp\u0027: \u00271778706413.00000\u0027, \u0027ETag\u0027: \u00270b4c12d7e0a73840c1c4f148fda3b037\u0027, \u0027X-Object-Sysmeta-Ec-Etag\u0027: \u0027same-ec-etag\u0027} (incoming)\n```\n\n^ i like those well enough!  ignore the typo that was me verifying the message in a test failure 😅\n\nthe amount of new code we have to read to *maintain* these messages is annoying, and the eventual \"confliciting pre-existing metadata: a \u003d\u003d b\" is going to be annoying when we have to decide if the problem was content-length, x-delete-at or user-metadata; but at least it\u0027s a start to see when ts \u003d\u003d ts \u0026 ec-etag !\u003d ec-etag","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d586fe9bb58c53c0be81ef621db9890c6ce14049","unresolved":true,"context_lines":[{"line_number":1997,"context_line":"        # requests to reference."},{"line_number":1998,"context_line":"        if self._tmppath:"},{"line_number":1999,"context_line":"            # It was a named temp file created by mkstemp()"},{"line_number":2000,"context_line":"            renamer(self._tmppath, target_path)"},{"line_number":2001,"context_line":"        else:"},{"line_number":2002,"context_line":"            self._safe_metadata_linkat(target_path, metadata)"},{"line_number":2003,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"a1b7d5aa_9d70a826","line":2000,"range":{"start_line":2000,"start_character":12,"end_line":2000,"end_character":47},"updated":"2026-05-13 05:38:00.000000000","message":"So if the file exists via O_TMPFILE (the else below) we check metadata and stop the collision.\nSo should we maybe do the same thing here? ie renamer could check for existence and check metadata before an os.rename? Heck we could even make it optional?\n\n```\nrenamer(..., check_metadata\u003dTrue)\n```","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c22919ecfe50a3ba569f81a6b9d33d3eee87d1ee","unresolved":true,"context_lines":[{"line_number":1997,"context_line":"        # requests to reference."},{"line_number":1998,"context_line":"        if self._tmppath:"},{"line_number":1999,"context_line":"            # It was a named temp file created by mkstemp()"},{"line_number":2000,"context_line":"            renamer(self._tmppath, target_path)"},{"line_number":2001,"context_line":"        else:"},{"line_number":2002,"context_line":"            self._safe_metadata_linkat(target_path, metadata)"},{"line_number":2003,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"bd17a862_d0294e1c","line":2000,"range":{"start_line":2000,"start_character":12,"end_line":2000,"end_character":47},"in_reply_to":"a1b7d5aa_9d70a826","updated":"2026-05-13 09:34:23.000000000","message":"\u003e check_metadata\u003dTrue\n\nYou mean add a new interface to that common helper?\n\n```\n$ ag \"def renamer\"\nswift/common/utils/__init__.py\n837:def renamer(old, new, fsync\u003dTrue):\n```\n\nI don\u0027t care about doing that; i don\u0027t know what environments would care about that.  I\u0027d caution against adding it because of the bloat and test churn - but if we just want bigger diffs I\u0027m sure codex would oblige!  \n\nThe difference with \"linkat -\u003e error -\u003e if matching metadata suppress\" is that the extra io/metadata-read is only in the uncommon error path; MOSTLY we just get to linkat the otmp and it works!\n\nIt\u0027s very efficient.  I think that\u0027s why basically all modern kernel/linux/fs have the o_tmpfile behavior since idk 2016 or w/e - i don\u0027t think you have to \"opt in\" - we do it if we can?\n\n... if we want to write more code for some hypothetical environment that might never get executed I\u0027d suggest it probably *would* have to be optional b/c the only way to have the same semantics with os.rename is LBYL which is probably racy.  Maybe the follow-up is \"require o_tmp\" or `use_slow_safe_rename\u003dTrue`\n\nO_TMPFILE really is better and it\u0027s been what everyone running swift has been using for like a decade or so.","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7703dfbb3ed2d7fdf8c47693120d9f8ef55826b8","unresolved":true,"context_lines":[{"line_number":92,"context_line":"COLLISION_METADATA_KEYS \u003d ("},{"line_number":93,"context_line":"    \u0027X-Timestamp\u0027,"},{"line_number":94,"context_line":"    \u0027ETag\u0027,"},{"line_number":95,"context_line":"    \u0027X-Object-Sysmeta-Ec-Etag\u0027,"},{"line_number":96,"context_line":")"},{"line_number":97,"context_line":"DATADIR_BASE \u003d \u0027objects\u0027"},{"line_number":98,"context_line":"ASYNCDIR_BASE \u003d \u0027async_pending\u0027"}],"source_content_type":"text/x-python","patch_set":18,"id":"8622dc09_c92d5d9f","line":95,"updated":"2026-05-13 22:05:14.000000000","message":"it will be easy to add x-delete-at or content-length to this list as follow-up if we decide that is useful; I think it\u0027s easier to start small and expand than start too broadly and have to pull back.","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":1985,"context_line":"            #    an error to prevent silent data corruption."},{"line_number":1986,"context_line":"            with open(target_path, \u0027rb\u0027) as f:"},{"line_number":1987,"context_line":"                existing_metadata \u003d read_metadata(f)"},{"line_number":1988,"context_line":"            if existing_metadata !\u003d metadata:"},{"line_number":1989,"context_line":"                emeta, meta \u003d ["},{"line_number":1990,"context_line":"                    _collision_metadata(m)"},{"line_number":1991,"context_line":"                    for m in (existing_metadata, metadata)]"}],"source_content_type":"text/x-python","patch_set":18,"id":"e5c1cb19_a67b33d9","line":1988,"updated":"2026-05-15 15:20:14.000000000","message":"ok, so the premise is that if metadata matches then, given metadata includes the etag, the contents also match","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a9b8fd54dedc735b04a493a4bdf8ccef3b15a53b","unresolved":false,"context_lines":[{"line_number":1985,"context_line":"            #    an error to prevent silent data corruption."},{"line_number":1986,"context_line":"            with open(target_path, \u0027rb\u0027) as f:"},{"line_number":1987,"context_line":"                existing_metadata \u003d read_metadata(f)"},{"line_number":1988,"context_line":"            if existing_metadata !\u003d metadata:"},{"line_number":1989,"context_line":"                emeta, meta \u003d ["},{"line_number":1990,"context_line":"                    _collision_metadata(m)"},{"line_number":1991,"context_line":"                    for m in (existing_metadata, metadata)]"}],"source_content_type":"text/x-python","patch_set":18,"id":"084d8162_077c07e9","line":1988,"in_reply_to":"e5c1cb19_a67b33d9","updated":"2026-06-01 09:55:11.000000000","message":"Done","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":2040,"context_line":""},{"line_number":2041,"context_line":"        # If rename is successful, flag put as succeeded. This is done to avoid"},{"line_number":2042,"context_line":"        # unnecessary os.unlink() of tempfile later. As renamer() has"},{"line_number":2043,"context_line":"        # succeeded, the tempfile would no longer exist at its original path."},{"line_number":2044,"context_line":"        self._put_succeeded \u003d True"},{"line_number":2045,"context_line":"        if cleanup:"},{"line_number":2046,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":18,"id":"da405833_d01fe07c","line":2043,"updated":"2026-05-15 15:20:14.000000000","message":"Claude told me:\n```\nStale comment after _put_succeeded \u003d True (swift/obj/diskfile.py:2044)\n  The comment \"If rename is successful\" no longer applies to the linkat branch where silent-EEXIST also counts as success. Should be updated to cover both\n  branches.\n```\n\nbut the comment was already \"stale\" in that the unlink only applies to when there is self._tmppath, so... \"OFF TOPIC\"\n\nFWIW ``self._put_succeeded`` is only relevant in that case, see https://review.opendev.org/c/openstack/swift/+/988782","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":2171,"context_line":"    :param etag_validate_frac: the probability that we should perform etag"},{"line_number":2172,"context_line":"                               validation during a complete file read"},{"line_number":2173,"context_line":"    \"\"\""},{"line_number":2174,"context_line":""},{"line_number":2175,"context_line":"    def __init__(self, fp, data_file, obj_size, etag,"},{"line_number":2176,"context_line":"                 disk_chunk_size, keep_cache_size, device_path, logger,"},{"line_number":2177,"context_line":"                 quarantine_hook, use_splice, pipe_size, diskfile,"}],"source_content_type":"text/x-python","patch_set":18,"id":"3f258992_474654f2","line":2174,"updated":"2026-05-15 15:20:14.000000000","message":":O seems adding these blank lines is becoming the fashion ;-)","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"}],"test/probe/test_timestamp_collision.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":39,"context_line":"        self.waiting \u003d None"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def track(self, req):"},{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"b5f23463_de645284","line":42,"updated":"2025-02-05 16:09:01.000000000","message":"it\u0027s a little hard to see; this is mapping the id of the request to an integer:\n\n0 is the first request\n1 is the second request\n\nThe zipper only ever supports two requests.\n\nthe setdefault returns (0, 1) and expects the caller to send that integer back in with each call to wait.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":39,"context_line":"        self.waiting \u003d None"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def track(self, req):"},{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"28f52f31_a0c3c015","line":42,"in_reply_to":"b5f23463_de645284","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"},{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"}],"source_content_type":"text/x-python","patch_set":3,"id":"d31c40a9_f447efb5","line":45,"updated":"2025-02-05 16:09:01.000000000","message":"req_id here is our tracking id; either (0, 1) so this is just calculating the \"other_id\"","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":42,"context_line":"        return self.tracking_ids.setdefault(id(req), len(self.tracking_ids))"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    def wait(self, req_id, fi):"},{"line_number":45,"context_line":"        my_id, other_id \u003d req_id, (req_id + 1) % 2"},{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"}],"source_content_type":"text/x-python","patch_set":3,"id":"54e1c9e4_81c89a1a","line":45,"in_reply_to":"d31c40a9_f447efb5","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"},{"line_number":49,"context_line":"            self.win[my_id] \u003d fi"},{"line_number":50,"context_line":"            self.loose[other_id] \u003d fi"},{"line_number":51,"context_line":"        else:"},{"line_number":52,"context_line":"            self.loose[my_id] \u003d fi"}],"source_content_type":"text/x-python","patch_set":3,"id":"ee3c9c8f_89cbf674","line":49,"updated":"2025-02-05 16:09:01.000000000","message":"this can\u0027t be what was intended; `self.win` was originally a list of two sets?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":46,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":47,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":48,"context_line":"            # i win this one!"},{"line_number":49,"context_line":"            self.win[my_id] \u003d fi"},{"line_number":50,"context_line":"            self.loose[other_id] \u003d fi"},{"line_number":51,"context_line":"        else:"},{"line_number":52,"context_line":"            self.loose[my_id] \u003d fi"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f6006b1_b9ef3450","line":49,"in_reply_to":"ee3c9c8f_89cbf674","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":195,"context_line":"        )[:-300000]  # last frag is empty, second to last is short"},{"line_number":196,"context_line":"        now \u003d Timestamp.now()"},{"line_number":197,"context_line":"        orig_headers \u003d {"},{"line_number":198,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":199,"context_line":"        }"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        original_transfer_data \u003d ECObjectController._transfer_data"}],"source_content_type":"text/x-python","patch_set":3,"id":"f1425e4c_039aca96","line":198,"updated":"2025-02-05 16:09:01.000000000","message":"\"merely\" doing two uploads with the same timestamp one after the other will consistently result in only a single encrypted-etag on disk.\n\nIf the first one finishes the second write will get 409s or maybe go to handoffs.\n\nIf the first one is still going when the second one starts; by time the second one finishes it will unconditionally call rename and blat out all of the first ones frags and overwrite *all* the encrypted etags from the first.\n\nThe only way \"two simultaneous writes\" can result in a mix of encrypted-etags is if after they manage to get a timestamp collision they also have their individual object server requests from each also finish inconsistently.  It\u0027s a fascinating distributed race that you only get to see at massive scale.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":195,"context_line":"        )[:-300000]  # last frag is empty, second to last is short"},{"line_number":196,"context_line":"        now \u003d Timestamp.now()"},{"line_number":197,"context_line":"        orig_headers \u003d {"},{"line_number":198,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":199,"context_line":"        }"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        original_transfer_data \u003d ECObjectController._transfer_data"}],"source_content_type":"text/x-python","patch_set":3,"id":"fda97c02_a2be5f63","line":198,"in_reply_to":"d75a204c_fb0ebd31","updated":"2026-05-13 03:24:34.000000000","message":"\u003e just make every proxy send a unique x-timestamp\n\ndamn skippy we can!  that\u0027s not this patch tho.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6dffbc7e7b97f29b3e90a8e06ff588d3626bdd21","unresolved":true,"context_lines":[{"line_number":195,"context_line":"        )[:-300000]  # last frag is empty, second to last is short"},{"line_number":196,"context_line":"        now \u003d Timestamp.now()"},{"line_number":197,"context_line":"        orig_headers \u003d {"},{"line_number":198,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":199,"context_line":"        }"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        original_transfer_data \u003d ECObjectController._transfer_data"}],"source_content_type":"text/x-python","patch_set":3,"id":"d75a204c_fb0ebd31","line":198,"in_reply_to":"f1425e4c_039aca96","updated":"2025-08-29 04:18:21.000000000","message":"Can we take a peice from the lamport clock algorithm and just make every proxy send a unique x-timestamp? Fact is, we already have the ability to add a counter to the timestamp, that\u0027s what the offset of a Timestamp object it. That is each proxy adds it\u0027s own unique offset to the timestamp.. then they\u0027ll never collide. But otherwise objects are always treated as ts.normal which is the right timestamp?\n\nBetter, use something like a unique offset + pid so each worker is unique.. going further if eventlet threads are also a case throw in some random number into the request env as it passes through our swifthttp class that can be used to unique out originating requests\n\n   x-timestmap \u003d Timestamp(\u003cts\u003e, offset\u003d\u003cuniq_id\u003e + \u003cpid\u003e + \u003creq number\u003e)\n\nA unique id could just be generated from the mac address or something, cause it isn\u0027t something I\u0027d like to define manually.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":207,"context_line":"            # annotate the putters with tracking_id based off req"},{"line_number":208,"context_line":"            tracking_id \u003d zipper.track(req)"},{"line_number":209,"context_line":"            for putter in putters:"},{"line_number":210,"context_line":"                putter.__tracking_id \u003d tracking_id"},{"line_number":211,"context_line":"            return original_transfer_data("},{"line_number":212,"context_line":"                controller, req, policy, data_source, putters, nodes,"},{"line_number":213,"context_line":"                min_conns, etag_hasher)"}],"source_content_type":"text/x-python","patch_set":3,"id":"517254a0_2d48abfd","line":210,"updated":"2025-02-05 16:09:01.000000000","message":"if the putter already has a req instance it doesn\u0027t really need to hold on to the tracking_id on behalf of the zipper?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            # annotate the putters with tracking_id based off req"},{"line_number":208,"context_line":"            tracking_id \u003d zipper.track(req)"},{"line_number":209,"context_line":"            for putter in putters:"},{"line_number":210,"context_line":"                putter.__tracking_id \u003d tracking_id"},{"line_number":211,"context_line":"            return original_transfer_data("},{"line_number":212,"context_line":"                controller, req, policy, data_source, putters, nodes,"},{"line_number":213,"context_line":"                min_conns, etag_hasher)"}],"source_content_type":"text/x-python","patch_set":3,"id":"03683b89_3933b07d","line":210,"in_reply_to":"517254a0_2d48abfd","updated":"2025-09-05 22:01:06.000000000","message":"Done","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":218,"context_line":"            fi \u003d footer_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027]"},{"line_number":219,"context_line":"            orig_end_of_object_data(putter, footer_metadata)"},{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"4eb83913_4d3493c7","line":221,"updated":"2025-02-05 16:09:01.000000000","message":"I can imagine the tracking_id is mabye more helpful than id(req) - but we could print zipper.tracking_id[putter.req]","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":218,"context_line":"            fi \u003d footer_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027]"},{"line_number":219,"context_line":"            orig_end_of_object_data(putter, footer_metadata)"},{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"27a782ec_264be01f","line":221,"in_reply_to":"4eb83913_4d3493c7","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"}],"source_content_type":"text/x-python","patch_set":3,"id":"b9f3fa5e_a0490bae","line":223,"updated":"2025-02-05 16:09:01.000000000","message":"the requirement of this interface to pass back the internal tracking_id seems weird.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":220,"context_line":"            sleep(0.3)  # let the obj server flush or link or something"},{"line_number":221,"context_line":"            print(putter.__tracking_id, fi,"},{"line_number":222,"context_line":"                  footer_metadata[\u0027X-Object-Sysmeta-Ec-Etag\u0027])"},{"line_number":223,"context_line":"            zipper.wait(putter.__tracking_id, fi)"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"}],"source_content_type":"text/x-python","patch_set":3,"id":"77316f16_189f9533","line":223,"in_reply_to":"b9f3fa5e_a0490bae","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"},{"line_number":227,"context_line":"                mock.patch.object(MIMEPutter, \u0027end_of_object_data\u0027,"},{"line_number":228,"context_line":"                                  patched_end_of_object_data):"},{"line_number":229,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":230,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":231,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"0c54728d_8940978b","line":228,"updated":"2025-02-05 16:09:01.000000000","message":"this patching isn\u0027t really changing what _transfer_data or end_of_object_data DO - just giving the test a hook to call `wait` in order to force the two `do_upload` streams to interleave such that we get the overlapping frags.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a75f814706c176bedaa9f8f2031c90bbc2bf495e","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        with mock.patch.object(ECObjectController, \u0027_transfer_data\u0027,"},{"line_number":226,"context_line":"                               patched_transfer_data), \\"},{"line_number":227,"context_line":"                mock.patch.object(MIMEPutter, \u0027end_of_object_data\u0027,"},{"line_number":228,"context_line":"                                  patched_end_of_object_data):"},{"line_number":229,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":230,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":231,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"69dc21ae_8ff8558f","line":228,"in_reply_to":"0c54728d_8940978b","updated":"2025-09-05 22:01:06.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"33faec0e_256f1487","line":248,"updated":"2025-02-05 16:09:01.000000000","message":"I think I\u0027d suppose the assertion might want to see is like how many frags have which different encrypted-etag?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6983434069772330db7c088366042a374f14e386","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"75cd9fcc_37a3df96","line":248,"in_reply_to":"33faec0e_256f1487","updated":"2025-08-12 22:42:48.000000000","message":"hey!  this tests still works to create overlapping frags!\n\n\n```\nvagrant@saio:~$ python3 /vagrant/.scratch/nv-swift-gizmos/bin/rebuild_frags.py 1755038107.63809.tar.gz \n{\n  \"1755038107.63809\": {\n    \"8e54a2b23489f9a635f5bbd278db659e\": {\n      \"count\": 2,\n      \"frags\": \"0,2\"\n    },\n    \"50d0313d43a3f6c65275a98d67f694d0\": {\n      \"count\": 4,\n      \"frags\": \"1,3,4,5\"\n    }\n  }\n}\nWARNING: discarding metadata ...\n{\n  \"X-Object-Meta-Foo\": \"bar\",\n  \"X-Object-Meta-Xtime\": \"t1\"\n}\nusing path: \u0027/AUTH_test/badtest/badnews\u0027\nusing metadata ... \n{\n  \"X-Object-Meta-Bar\": \"baz\",\n  \"X-Object-Meta-Xtime\": \"t2\"\n}\n0 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 262144\n1 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 524288\n2 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 786432\n3 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 1048576\n0 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 1310720\n1 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 1572864\n2 8e54a2b23489f9a635f5bbd278db659e provided 262144 / 1835008\n3 50d0313d43a3f6c65275a98d67f694d0 provided 262144 / 2097152\n0 8e54a2b23489f9a635f5bbd278db659e provided 187144 / 2284296\n1 50d0313d43a3f6c65275a98d67f694d0 provided 187144 / 2471440\n2 8e54a2b23489f9a635f5bbd278db659e provided 187144 / 2658584\n3 50d0313d43a3f6c65275a98d67f694d0 provided 187144 / 2845728\nexpected 4c38d6ba3d586810e10603df2047a847 \u003d\u003d 4c38d6ba3d586810e10603df2047a847 2845728\n```\n\n... what\u0027s crazy is this is actually a reasonable approximation of what we observe in prod.  Except both uploads have the same user metadata too - just different crypto meta.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5830842fc254dfd922e52778fae5bd7d457d7aec","unresolved":true,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfd8dbdb_ff5694ec","line":248,"in_reply_to":"75cd9fcc_37a3df96","updated":"2025-09-09 02:56:38.000000000","message":"yes, when I turned on encryption within my vasio, I saw proxy-server running into \"ETag mismatch\":\n```\nSep  8 21:53:45 saio proxy-server: Problem with fragment response: ETag mismatch (txn: tx05ab7dff95254a8287878-0068bf5069) (client_ip: 127.0.0.1)\n```\n\nand if I read the object in the end of probe test, ``test_overlap_data_write_streams`` failed because request returned 503.\n```\nFAILED swift/test/probe/test_timestamp_collision.py::TestECCollision::test_overlap_data_write_streams - swift.common.internal_client.UnexpectedResponse: Unexpected response: 503 Service Unavailable (b\u0027\u003chtml\u003e\u003ch...\n```","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":245,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":246,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":247,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":248,"context_line":"        self.assertTrue(metadata)"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":251,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":3,"id":"165836fd_25f6e2be","line":248,"in_reply_to":"bfd8dbdb_ff5694ec","updated":"2025-09-19 23:19:58.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":270,"context_line":"        # re-upload with same timestamp (add some metadata)"},{"line_number":271,"context_line":"        headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"}],"source_content_type":"text/x-python","patch_set":3,"id":"6235cb1a_337c1930","line":273,"updated":"2025-02-05 16:09:01.000000000","message":"i\u0027m not sure what this test is trying to prove; obviously the \"overlap\" is going to write to handoffs if the primary disk is down?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":270,"context_line":"        # re-upload with same timestamp (add some metadata)"},{"line_number":271,"context_line":"        headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"}],"source_content_type":"text/x-python","patch_set":3,"id":"13255852_acb5430c","line":273,"in_reply_to":"6235cb1a_337c1930","updated":"2026-05-13 03:24:34.000000000","message":"ah, the trick is it\u0027s setting the timestamp; so end the end you get an unstable set of writes.  I guess that is a little on the nose; but it maybe offers a framework for setting up \"same timestamp with different metadata\"","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":277,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":278,"context_line":"        # but the save_file metadata presists!"}],"source_content_type":"text/x-python","patch_set":3,"id":"229d19b4_5c0a9878","line":275,"updated":"2025-02-05 16:09:01.000000000","message":"right, cause we used internal client to force the timestamp collision, and the encrypted etag doesn\u0027t effect the df path/name.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":272,"context_line":"        self.do_upload(contents, headers)"},{"line_number":273,"context_line":"        self.revive_drive(save_device)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":276,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":277,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":278,"context_line":"        # but the save_file metadata presists!"}],"source_content_type":"text/x-python","patch_set":3,"id":"7ae0ea81_10eca397","line":275,"in_reply_to":"229d19b4_5c0a9878","updated":"2026-05-13 03:24:34.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":279,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":280,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"f1b7777b_06b25af6","line":282,"updated":"2025-02-05 16:09:01.000000000","message":"these are such weird/specific assertions; maybe better as a `read_meta_from_files` helper and then you can:\n\n```\nnew_files \u003d []\norig_meta \u003d None\nfor f in df2node:\n    if f \u003d\u003d save_file:\n        orig_meta \u003d self.read_meta_from_files([f])\n    else:\n        new_files.append(f)\nself.assertEqual(1, len(orig_meta))\nself.assertNotIn(\u0027foo\u0027, orig_meta[0])\nnew_meta \u003d self.read_meta_from_files(new_files)\nself.assertFalse(any(\u0027foo\u0027 in meta for meta in new_meta), \u0027found foo in %r\u0027 % new_meta)\n```","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":279,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":280,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"6c9fa4f4_bf7dc444","line":282,"in_reply_to":"f1b7777b_06b25af6","updated":"2025-09-19 23:19:58.000000000","message":"Acknowledged","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f64bb79b62dafa5b119011bd3c909705f343b75c","unresolved":true,"context_lines":[{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("},{"line_number":286,"context_line":"            self.account, self.container_name, self.object_name)"},{"line_number":287,"context_line":"        self.assertEqual(contents, b\u0027\u0027.join(body_iter))"}],"source_content_type":"text/x-python","patch_set":3,"id":"c9984fee_062a7edb","line":284,"updated":"2025-02-05 16:09:01.000000000","message":"it might be more interesting if we saved nparity+1 of the orig files - then ran the reconstructor to swallow the handoffs and then download probably wouldn\u0027t work?","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":281,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"},{"line_number":282,"context_line":"                                         f !\u003d save_file], \u0027foo\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        # download still works tho"},{"line_number":285,"context_line":"        status, headers, body_iter \u003d self.swift.get_object("},{"line_number":286,"context_line":"            self.account, self.container_name, self.object_name)"},{"line_number":287,"context_line":"        self.assertEqual(contents, b\u0027\u0027.join(body_iter))"}],"source_content_type":"text/x-python","patch_set":3,"id":"17f40570_8ce8a873","line":284,"in_reply_to":"c9984fee_062a7edb","updated":"2026-05-13 03:24:34.000000000","message":"that would definately be more interesting.\n\n... and on a 4+2 with 8 disks we just *barely* have enough room to save 3 \"bad\" primaries and write to 5 disks w/ handoffs.","commit_id":"182f85a2353528332911b72dff40d3f854dcaf80"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2010-2024 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":8,"id":"417b9529_9ffc2594","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":25},"updated":"2025-09-19 14:20:21.000000000","message":"it\u0027s 2025 ;-)","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2010-2024 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":8,"id":"eaf19e64_cdf43cee","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":25},"in_reply_to":"417b9529_9ffc2594","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":51,"context_line":"        # Two sets tracking which fragment indexes each request has sent."},{"line_number":52,"context_line":"        self.sent \u003d [set(), set()]"},{"line_number":53,"context_line":"        # Two sets tracking fragments where this request \"lost\""},{"line_number":54,"context_line":"        self.loose \u003d [set(), set()]"},{"line_number":55,"context_line":"        # Two sets tracking fragments where this request \"won\""},{"line_number":56,"context_line":"        self.win \u003d [set(), set()]"},{"line_number":57,"context_line":"        self.waiting \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"28f876b7_b052ead8","line":54,"range":{"start_line":54,"start_character":13,"end_line":54,"end_character":18},"updated":"2025-09-19 14:20:21.000000000","message":"s/loose/lose/\n\n\"loose\" has a different meaning","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e35973f65093e5d9d73178f6563976f8ad026eb4","unresolved":false,"context_lines":[{"line_number":51,"context_line":"        # Two sets tracking which fragment indexes each request has sent."},{"line_number":52,"context_line":"        self.sent \u003d [set(), set()]"},{"line_number":53,"context_line":"        # Two sets tracking fragments where this request \"lost\""},{"line_number":54,"context_line":"        self.loose \u003d [set(), set()]"},{"line_number":55,"context_line":"        # Two sets tracking fragments where this request \"won\""},{"line_number":56,"context_line":"        self.win \u003d [set(), set()]"},{"line_number":57,"context_line":"        self.waiting \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"25076a4b_cc892074","line":54,"range":{"start_line":54,"start_character":13,"end_line":54,"end_character":18},"in_reply_to":"28f876b7_b052ead8","updated":"2025-09-25 06:14:18.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":76,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":79,"context_line":"            # request that sent this fragment wins this fragment"},{"line_number":80,"context_line":"            self.win[my_id].add(fi)"},{"line_number":81,"context_line":"            self.loose[other_id].add(fi)"},{"line_number":82,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"66917782_86f9c187","line":79,"updated":"2025-09-19 14:20:21.000000000","message":"``if fi in self.sent[other_id]`` implies the other request has already sent this frag, so why does this clause then proceed to record *this* request as the winner and the other as the loser?\n\nIt probably makes no difference to the logic but it seems that the naming has been inverted such that self.win is recording the losing frags??","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":79,"context_line":"            # request that sent this fragment wins this fragment"},{"line_number":80,"context_line":"            self.win[my_id].add(fi)"},{"line_number":81,"context_line":"            self.loose[other_id].add(fi)"},{"line_number":82,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"f0f20b63_71605e29","line":79,"in_reply_to":"0f74781a_e8f5e53c","updated":"2026-05-13 03:24:34.000000000","message":"oh wait, now I see it?  so maybe this is equivilent?\n\n```\ndiff --git a/test/probe/test_timestamp_collision.py b/test/probe/test_timestamp_collision.py\nindex edb2eb9c83..f25157da9b 100644\n--- a/test/probe/test_timestamp_collision.py\n+++ b/test/probe/test_timestamp_collision.py\n@@ -79,11 +79,11 @@ class FragZipper(object):\n \n         if fi in self.sent[other_id]:\n             # request that sent this fragment wins this fragment\n-            self.win[my_id].add(fi)\n-            self.lose[other_id].add(fi)\n-        else:\n             self.lose[my_id].add(fi)\n             self.win[other_id].add(fi)\n+        else:\n+            self.win[my_id].add(fi)\n+            self.lose[other_id].add(fi)\n \n         # check if request should wait\n         if (len(self.win[my_id]) \u003e len(self.win[other_id]) or\n```","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":true,"context_lines":[{"line_number":76,"context_line":"        self.sent[my_id].add(fi)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"        if fi in self.sent[other_id]:"},{"line_number":79,"context_line":"            # request that sent this fragment wins this fragment"},{"line_number":80,"context_line":"            self.win[my_id].add(fi)"},{"line_number":81,"context_line":"            self.loose[other_id].add(fi)"},{"line_number":82,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"0f74781a_e8f5e53c","line":79,"in_reply_to":"66917782_86f9c187","updated":"2025-09-19 21:27:59.000000000","message":"on master the behavior was \"last write wins\" - I think we the error on link_at we invert the \"winner\"","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"        # check if request should wait"},{"line_number":87,"context_line":"        if (len(self.win[my_id]) \u003e len(self.win[other_id]) or"},{"line_number":88,"context_line":"                len(self.loose[my_id]) \u003e len(self.loose[other_id])):"},{"line_number":89,"context_line":"            if not self.waiting:  # prevent both requests from waiting"},{"line_number":90,"context_line":"                e \u003d self.waiting \u003d event.Event()"},{"line_number":91,"context_line":"                print(my_id, \u0027waiting\u0027, self.sent)"}],"source_content_type":"text/x-python","patch_set":8,"id":"62e80766_71d2dad6","line":88,"range":{"start_line":88,"start_character":16,"end_line":88,"end_character":66},"updated":"2025-09-19 14:20:21.000000000","message":"I\u0027m failing to understand this condition: if my_id has lost more times than   other_id, why does it now wait and presumably lose again?\n\nI also don\u0027t really understand the significance of win vs lose: isn\u0027t is sufficient for one thread to wait if it has sent more frags than the other?\n\nI changes this condition to \n\n```\nif len(self.sent[my_id]) \u003e len(self.sent[other_id]):\n```\n and the test still passes (I ran it \u003e\u003e10 times).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"        # check if request should wait"},{"line_number":87,"context_line":"        if (len(self.win[my_id]) \u003e len(self.win[other_id]) or"},{"line_number":88,"context_line":"                len(self.loose[my_id]) \u003e len(self.loose[other_id])):"},{"line_number":89,"context_line":"            if not self.waiting:  # prevent both requests from waiting"},{"line_number":90,"context_line":"                e \u003d self.waiting \u003d event.Event()"},{"line_number":91,"context_line":"                print(my_id, \u0027waiting\u0027, self.sent)"}],"source_content_type":"text/x-python","patch_set":8,"id":"35a13914_c4c3f3cb","line":88,"range":{"start_line":88,"start_character":16,"end_line":88,"end_character":66},"in_reply_to":"62e80766_71d2dad6","updated":"2026-05-13 03:24:34.000000000","message":"WOMM is good enough to me - I was *flailing* to get this working *at all* and was SO happy when I *finally* had mixed etags in my EC timestamp collisions through what was ostensibly \"just\" some ~unmodified \"concurrent proxy writes\" that I probably gave up on understanding all the cruft and went back to hacking on my frag-fix-it tools!\n\nI repeated your test and agree `len(self.sent[my_id]) \u003e len(self.sent[other_id])` is roughly got to be short-hand for \"i\u0027m winning or loosing but not tied\" ???","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":8,"id":"03d27542_8ca75767","line":111,"range":{"start_line":111,"start_character":61,"end_line":111,"end_character":62},"updated":"2025-09-19 14:20:21.000000000","message":"is there a reason to want 3 attempts per request? I don\u0027t think we want the test to hide failed requests - see my comment at line 195","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":8,"id":"abb2e9f2_65553416","line":111,"range":{"start_line":111,"start_character":61,"end_line":111,"end_character":62},"in_reply_to":"03d27542_8ca75767","updated":"2025-09-19 21:27:59.000000000","message":"yeah, no that was wrong.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                    file_2_key[f].append(key)"},{"line_number":155,"context_line":"        return file_2_key"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def assertDatafilesMissingMetaKey(self, data_files, key):"},{"line_number":158,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":159,"context_line":"        errs \u003d defaultdict(list)"},{"line_number":160,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"08e6d0de_57384534","line":157,"updated":"2025-09-19 14:20:21.000000000","message":"these assertion helpers will pass if ``data_files`` is empty","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":154,"context_line":"                    file_2_key[f].append(key)"},{"line_number":155,"context_line":"        return file_2_key"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def assertDatafilesMissingMetaKey(self, data_files, key):"},{"line_number":158,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":159,"context_line":"        errs \u003d defaultdict(list)"},{"line_number":160,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3a891c8f_cc831496","line":157,"in_reply_to":"08e6d0de_57384534","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"4f51a2bd_065f1364","line":168,"updated":"2025-09-19 14:20:21.000000000","message":"nit (emphasize nit!): maybe follow the unittest naming pattern for containment assertions e.g. ``assertInDatafileMeta``, ``assertNotInDatafileMeta``\n\nor, another suggestion, just use a helper to gather the metadata dicts:\n\n```\nlist_of_dicts \u003d gather_datafile_metadata(data_files)\nassertTrue(all(key in d for d in list_of_dicts))\n```\n\nsee also line 295","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"fa00459c_e6498718","line":168,"in_reply_to":"1deb0afd_01ca1db2","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if errs:"},{"line_number":166,"context_line":"            self.fail(\u0027Some metadata for %s found on files: %r\u0027 % (key, errs))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def assertDatafilesHaveMetaKey(self, data_files, key):"},{"line_number":169,"context_line":"        meta_keys \u003d self._meta_keys(key)"},{"line_number":170,"context_line":"        errs \u003d []"},{"line_number":171,"context_line":"        for f in data_files:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1deb0afd_01ca1db2","line":168,"in_reply_to":"4f51a2bd_065f1364","updated":"2025-09-19 21:27:59.000000000","message":"haha, yeah I think there\u0027s probably a better way to do this!","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"        orig_send_commit_confirmation \u003d MIMEPutter.send_commit_confirmation"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"}],"source_content_type":"text/x-python","patch_set":8,"id":"44df7670_79d1093f","line":192,"updated":"2025-09-19 14:20:21.000000000","message":"Nice. I\u0027m encouraged to see this mocking of internal client working to test concurrency because I think I\u0027ll need to do this kind of thing for native mpu 😊 Perhaps we already do it elsewhere and I\u0027m just not familiar.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"        orig_send_commit_confirmation \u003d MIMEPutter.send_commit_confirmation"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"}],"source_content_type":"text/x-python","patch_set":8,"id":"7a1a199a_0469442c","line":192,"in_reply_to":"44df7670_79d1093f","updated":"2025-09-19 21:27:59.000000000","message":"yeah I don\u0027t know where all we do this - i\u0027ve done it before to a pached internal-client to make POST requests to data files in the \"wrong\" storage policy index.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"},{"line_number":196,"context_line":"            return orig_send_commit_confirmation(putter)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"        with mock.patch.object(MIMEPutter, \u0027send_commit_confirmation\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"816cbb89_1b3e61a6","line":195,"updated":"2025-09-19 14:20:21.000000000","message":"I was playing with the test and wanted to check that the test would fail if the first request (the one that waits for proceed) failed. So I raised an exception right here, and the test still passed! I figured out that was because the internal client retried the request and the retry does not enter this if clause. But I think I\u0027d prefer the request to be asserting that the *first attempt* to commit the paused racing request *does succeed*, so I\u0027d suggest the the InternalClient should be configured to make only one attempt (and with that change my exception caused the test to fail as expected).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":192,"context_line":"        def patched_send_commit_confirmation(putter):"},{"line_number":193,"context_line":"            if not ready.ready():"},{"line_number":194,"context_line":"                ready.send(None)"},{"line_number":195,"context_line":"                proceed.wait()"},{"line_number":196,"context_line":"            return orig_send_commit_confirmation(putter)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"        with mock.patch.object(MIMEPutter, \u0027send_commit_confirmation\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"16ace967_ad8f1b68","line":195,"in_reply_to":"816cbb89_1b3e61a6","updated":"2025-09-19 21:27:59.000000000","message":"yeah i agree - retries is the default and not helpful at all in this patched test context.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":202,"context_line":"            gt \u003d spawn(self.do_upload, contents, headers)"},{"line_number":203,"context_line":"            # let the first thread run up to commit"},{"line_number":204,"context_line":"            ready.wait()"},{"line_number":205,"context_line":"            # we should have a bunch of non-durable"},{"line_number":206,"context_line":"            orig_df2node \u003d self.map_data_file_to_node()"},{"line_number":207,"context_line":"            self.assertDatafilesHaveMetaKey(orig_df2node, \u0027foo\u0027)"},{"line_number":208,"context_line":"            # overwrite from another thread"}],"source_content_type":"text/x-python","patch_set":8,"id":"3d205d04_8f486f8a","line":205,"range":{"start_line":205,"start_character":31,"end_line":205,"end_character":51},"updated":"2025-09-19 14:20:21.000000000","message":"that should be asserted - I don\u0027t think there\u0027s any assertion that verifies there are *any* files","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":202,"context_line":"            gt \u003d spawn(self.do_upload, contents, headers)"},{"line_number":203,"context_line":"            # let the first thread run up to commit"},{"line_number":204,"context_line":"            ready.wait()"},{"line_number":205,"context_line":"            # we should have a bunch of non-durable"},{"line_number":206,"context_line":"            orig_df2node \u003d self.map_data_file_to_node()"},{"line_number":207,"context_line":"            self.assertDatafilesHaveMetaKey(orig_df2node, \u0027foo\u0027)"},{"line_number":208,"context_line":"            # overwrite from another thread"}],"source_content_type":"text/x-python","patch_set":8,"id":"4917c460_b2b77611","line":205,"range":{"start_line":205,"start_character":31,"end_line":205,"end_character":51},"in_reply_to":"3d205d04_8f486f8a","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":209,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":210,"context_line":"            headers[\u0027x-object-meta-bar\u0027] \u003d \u0027baz\u0027"},{"line_number":211,"context_line":"            self.do_upload(contents, headers)"},{"line_number":212,"context_line":"            # they should all be durable"},{"line_number":213,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":214,"context_line":"            self.assertDatafilesMissingMetaKey(df2node, \u0027foo\u0027)"},{"line_number":215,"context_line":"            self.assertDatafilesHaveMetaKey(df2node, \u0027bar\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"28474cff_d9bb4123","line":212,"updated":"2025-09-19 14:20:21.000000000","message":"ditto, it would be good to actually assert the on disk files are as expected","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":209,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":210,"context_line":"            headers[\u0027x-object-meta-bar\u0027] \u003d \u0027baz\u0027"},{"line_number":211,"context_line":"            self.do_upload(contents, headers)"},{"line_number":212,"context_line":"            # they should all be durable"},{"line_number":213,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":214,"context_line":"            self.assertDatafilesMissingMetaKey(df2node, \u0027foo\u0027)"},{"line_number":215,"context_line":"            self.assertDatafilesHaveMetaKey(df2node, \u0027bar\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"951eb603_3a913bf4","line":212,"in_reply_to":"28474cff_d9bb4123","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":263,"context_line":"                                  patched_end_of_object_data):"},{"line_number":264,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":265,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":266,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"},{"line_number":267,"context_line":"            gt1 \u003d spawn(self.do_upload, contents, headers)"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"            headers \u003d dict(orig_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"79ca8f9a_e9f114e7","line":266,"updated":"2025-09-19 14:20:21.000000000","message":"AFAICT these xtime metadata are not asserted, and the use of ``t1`` and ``t2`` values is a little distracting given that the requests both have the same timestamp.","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":263,"context_line":"                                  patched_end_of_object_data):"},{"line_number":264,"context_line":"            headers \u003d dict(orig_headers)"},{"line_number":265,"context_line":"            headers[\u0027x-object-meta-foo\u0027] \u003d \u0027bar\u0027"},{"line_number":266,"context_line":"            headers[\u0027x-object-meta-xtime\u0027] \u003d \u0027t1\u0027"},{"line_number":267,"context_line":"            gt1 \u003d spawn(self.do_upload, contents, headers)"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"            headers \u003d dict(orig_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"0ed840f6_c3122e25","line":266,"in_reply_to":"79ca8f9a_e9f114e7","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":278,"context_line":"            except Timeout:"},{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"}],"source_content_type":"text/x-python","patch_set":8,"id":"6bbfee55_9d04c31e","line":281,"updated":"2025-09-19 14:20:21.000000000","message":"it would be good to assert there are durable data files","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            except Timeout:"},{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"}],"source_content_type":"text/x-python","patch_set":8,"id":"32bbecdb_be52a332","line":281,"in_reply_to":"6bbfee55_9d04c31e","updated":"2025-09-19 21:27:59.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"}],"source_content_type":"text/x-python","patch_set":8,"id":"306a429a_7bbdc7ca","line":282,"updated":"2025-09-19 14:20:21.000000000","message":"the previous 2 lines cam be de-dented - I like to exit mock\u0027s ASAP","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                self.fail(\u0027probably deadlock because of bugs \u0027"},{"line_number":280,"context_line":"                          \u0027(or huge contents?); check logs\u0027)"},{"line_number":281,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"}],"source_content_type":"text/x-python","patch_set":8,"id":"0512e091_f70ae491","line":282,"in_reply_to":"306a429a_7bbdc7ca","updated":"2025-09-19 21:27:59.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"},{"line_number":286,"context_line":"        # Check that metadata for \u0027foo\u0027 exists, either encrypted or not"},{"line_number":287,"context_line":"        foo_keys \u003d self._meta_keys(\u0027foo\u0027)"},{"line_number":288,"context_line":"        self.assertTrue(any(key in all_found_keys for key in foo_keys),"}],"source_content_type":"text/x-python","patch_set":8,"id":"0106eb4f_18ca8610","line":285,"updated":"2025-09-19 14:20:21.000000000","message":"this just reduces to \n\n```\nall_keys \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027]).values()\n        all_found_keys \u003d [key for keys in all_keys for key in keys]\n```\n\nor make capture_metadata return the flattened list of all keys?","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":282,"context_line":"            metadata \u003d self.capture_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"        all_keys \u003d [key for key in metadata.values()]"},{"line_number":285,"context_line":"        all_found_keys \u003d [key for keys in all_keys for key in keys]"},{"line_number":286,"context_line":"        # Check that metadata for \u0027foo\u0027 exists, either encrypted or not"},{"line_number":287,"context_line":"        foo_keys \u003d self._meta_keys(\u0027foo\u0027)"},{"line_number":288,"context_line":"        self.assertTrue(any(key in all_found_keys for key in foo_keys),"}],"source_content_type":"text/x-python","patch_set":8,"id":"9faf0ed1_c2969fa6","line":285,"in_reply_to":"0106eb4f_18ca8610","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":292,"context_line":"        bar_keys \u003d self._meta_keys(\u0027bar\u0027)"},{"line_number":293,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":294,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":295,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":298,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":8,"id":"78464ffe_89cc5080","line":295,"updated":"2025-09-19 14:20:21.000000000","message":"this is similar to the assertDatafiles[Missing|Have]MetaKey, but uses a different pattern - it would be nice to unify the patterns, perhaps by having one helper to gather all the relevant metadata and then make assertions like this one (i.e. line 293, using ``any`` or ``all`` as appropriate).","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":false,"context_lines":[{"line_number":292,"context_line":"        bar_keys \u003d self._meta_keys(\u0027bar\u0027)"},{"line_number":293,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":294,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":295,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":298,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":8,"id":"ffa610eb_3e7b1d53","line":295,"in_reply_to":"78464ffe_89cc5080","updated":"2025-09-19 23:19:58.000000000","message":"Done","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c1bb7b43a5f243f412b097c32f0538c43ec7adf9","unresolved":true,"context_lines":[{"line_number":322,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":323,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":324,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":325,"context_line":"        # but the save_file metadata presists!"},{"line_number":326,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":327,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":328,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"}],"source_content_type":"text/x-python","patch_set":8,"id":"92bc82c6_6daa1827","line":325,"range":{"start_line":325,"start_character":37,"end_line":325,"end_character":45},"updated":"2025-09-19 14:20:21.000000000","message":"1) typo: persists\n\n2) it\u0027s not just that the *metadata* persists, it\u0027s that the *data file* persists because the node it was on has been offline during the overwrite","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7142b4554c973bf0259f4facf6810f2d74c7b59b","unresolved":false,"context_lines":[{"line_number":322,"context_line":"        # files on disk look exactly the same (!!)"},{"line_number":323,"context_line":"        new_df2node \u003d self.map_data_file_to_node()"},{"line_number":324,"context_line":"        self.assertEqual(df2node, new_df2node)"},{"line_number":325,"context_line":"        # but the save_file metadata presists!"},{"line_number":326,"context_line":"        self.assertDatafilesMissingMetaKey([f for f in new_df2node"},{"line_number":327,"context_line":"                                            if f \u003d\u003d save_file], \u0027foo\u0027)"},{"line_number":328,"context_line":"        self.assertDatafilesHaveMetaKey([f for f in new_df2node if"}],"source_content_type":"text/x-python","patch_set":8,"id":"95248093_7fca1926","line":325,"range":{"start_line":325,"start_character":37,"end_line":325,"end_character":45},"in_reply_to":"92bc82c6_6daa1827","updated":"2025-09-19 21:27:59.000000000","message":"Acknowledged","commit_id":"e5559f7ed00e330cee6aea02d4e1fbc8c33ba98c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":true,"context_lines":[{"line_number":281,"context_line":""},{"line_number":282,"context_line":"        def safe_upload(contents, headers):"},{"line_number":283,"context_line":"            try:"},{"line_number":284,"context_line":"                resp \u003d self.do_upload(contents, headers)"},{"line_number":285,"context_line":"            except internal_client.UnexpectedResponse as e:"},{"line_number":286,"context_line":"                resp \u003d e.resp"},{"line_number":287,"context_line":"            results.append(resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"e24cd9d4_5034c4de","line":284,"updated":"2025-09-23 15:23:02.000000000","message":"self.do_upload doesn\u0027t return anything, nor does self.swift.upload_object 😞","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":281,"context_line":""},{"line_number":282,"context_line":"        def safe_upload(contents, headers):"},{"line_number":283,"context_line":"            try:"},{"line_number":284,"context_line":"                resp \u003d self.do_upload(contents, headers)"},{"line_number":285,"context_line":"            except internal_client.UnexpectedResponse as e:"},{"line_number":286,"context_line":"                resp \u003d e.resp"},{"line_number":287,"context_line":"            results.append(resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"015b1d4c_925fea36","line":284,"in_reply_to":"e24cd9d4_5034c4de","updated":"2026-05-13 03:24:34.000000000","message":"ORLY!?\n\nlooks like we ONLY expect the Unexpected\n\n... better as a wrapper over `self.assertRaises`\n\noh, except then if you revert you get early failure:\n\n```\nAssertionError: UnexpectedResponse not raised\n```\n\nwelp, I\u0027m going with a comment:\n\n```\nexcept internal_client.UnexpectedResponse as e:\n    results.append(e.resp.status_int)\nelse:\n# do_upload wraps InternalClient.upload_object which returns\n# None on success; which only happens if you revert the change.\n# We leave the unexpected sentinal here instead of assertRaises\n# so it\u0027s easier to see the problematic results w/o this fix.\nresults.append(True)\n```\n\nand the nice assertion failure:\n\n```\n        # nothing is durable\n\u003e       self.assertEqual(0, len(self._collect_durable_files(df2node)), counts)\nE       AssertionError: 0 !\u003d 6 : defaultdict(\u003cclass \u0027int\u0027\u003e, {\u0027bar\u0027: 3, \u0027foo\u0027: 3})\n```\n\ndespite the unfortunate results:\n\n```\n# both responses were errors\nself.assertEqual([503, 503], results)\n\n(Pdb) !results\n[True, True]\n```","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1cdc094c088c52c15e580663efac7307e702adb9","unresolved":true,"context_lines":[{"line_number":309,"context_line":"        df2node \u003d self.map_data_file_to_node()"},{"line_number":310,"context_line":"        self.assertEqual(self.policy.ec_n_unique_fragments, len(df2node))"},{"line_number":311,"context_line":"        # nothing is durable"},{"line_number":312,"context_line":"        self.assertEqual(0, len(self._collect_durable_files(df2node)))"},{"line_number":313,"context_line":"        # both responses were errors"},{"line_number":314,"context_line":"        self.assertEqual([503, 503], [r.status_int for r in results])"},{"line_number":315,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"4c98a012_a5b52303","line":312,"updated":"2025-09-23 15:23:02.000000000","message":"I reverted utils to master and as expected this fails; I made a dif to be totally sure the zipper was mixing up the metadata when the files were all durable:\n\n```\ndiff --git a/test/probe/test_timestamp_collision.py b/test/probe/test_timestamp_collision.py\nindex 6c32d290c..c8c0c40f8 100644\n--- a/test/probe/test_timestamp_collision.py\n+++ b/test/probe/test_timestamp_collision.py\n@@ -308,12 +308,7 @@ class TestECCollision(ECProbeTest):\n \n         df2node \u003d self.map_data_file_to_node()\n         self.assertEqual(self.policy.ec_n_unique_fragments, len(df2node))\n-        # nothing is durable\n-        self.assertEqual(0, len(self._collect_durable_files(df2node)))\n-        # both responses were errors\n-        self.assertEqual([503, 503], [r.status_int for r in results])\n \n-        # metadata is all mixed up!\n         counts \u003d defaultdict(int)\n         metadata \u003d self._collect_datafile_metadata(df2node, [\u0027foo\u0027, \u0027bar\u0027])\n         foo_keys \u003d self._meta_keys(\u0027foo\u0027)\n@@ -326,6 +321,12 @@ class TestECCollision(ECProbeTest):\n                 self.assertTrue(any(key in m for key in bar_keys))\n                 counts[\u0027bar\u0027] +\u003d 1\n                 self.assertFalse(any(key in m for key in foo_keys))\n+\n+        # nothing is durable\n+        self.assertEqual(0, len(self._collect_durable_files(df2node)), counts)\n+        # both responses were errors\n+        self.assertEqual([503, 503], [r.status_int for r in results])\n+        # metadata is all mixed up!\n         # OMM I get all of these scenarios!?  I wonder if there\u0027s a way to make\n         # this more stable?\n         self.assertIn(counts, [\n\n```\n\nso I see\n\n```\n# nothing is durable\n\u003e       self.assertEqual(0, len(self._collect_durable_files(df2node)), counts)\nE       AssertionError: 0 !\u003d 6 : defaultdict(\u003cclass \u0027int\u0027\u003e, {\u0027foo\u0027: 3, \u0027bar\u0027: 3})\n```","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":309,"context_line":"        df2node \u003d self.map_data_file_to_node()"},{"line_number":310,"context_line":"        self.assertEqual(self.policy.ec_n_unique_fragments, len(df2node))"},{"line_number":311,"context_line":"        # nothing is durable"},{"line_number":312,"context_line":"        self.assertEqual(0, len(self._collect_durable_files(df2node)))"},{"line_number":313,"context_line":"        # both responses were errors"},{"line_number":314,"context_line":"        self.assertEqual([503, 503], [r.status_int for r in results])"},{"line_number":315,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3c195846_76a9d539","line":312,"in_reply_to":"4c98a012_a5b52303","updated":"2026-05-13 03:24:34.000000000","message":"Done","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"18771a9ace76eaeb7be1d75ff6e94cc17eb68314","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c7998f61_dff0dca5","line":377,"updated":"2025-09-19 23:19:58.000000000","message":"I think there\u0027s another df out on the hand off - might be interesting to throw some reconstructor into the mix here.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"5f3f65da_6631fda8","line":377,"in_reply_to":"c7998f61_dff0dca5","updated":"2026-05-13 03:24:34.000000000","message":"\u003e throw some reconstructor into the mix here.\n\nthis is a ReplProbeTest\n\nbut I don\u0027t see any good reason not to look at all the disks, or run the replicator to prove it can\u0027t fix anything.","commit_id":"c14a2e46b2a676529bf4fbd168831d8a66782b87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":false,"context_lines":[{"line_number":110,"context_line":"        self.object_name \u003d self.object_name.decode(\u0027utf8\u0027)"},{"line_number":111,"context_line":"        # XXX better to use random names each test; hit different parts"},{"line_number":112,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":113,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":114,"context_line":"        self.swift \u003d internal_client.InternalClient("},{"line_number":115,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 1)"},{"line_number":116,"context_line":"        self.swift.create_container("}],"source_content_type":"text/x-python","patch_set":15,"id":"b8a36292_d332d0fb","line":113,"updated":"2026-05-13 03:24:34.000000000","message":"no comments here!?","commit_id":"cf1537b6af35ec9277e3d254f39afd38ae578070"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":true,"context_lines":[{"line_number":462,"context_line":"        now \u003d Timestamp.now()"},{"line_number":463,"context_line":"        base_headers \u003d {"},{"line_number":464,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":465,"context_line":"        }"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"        num_sent \u003d 0"},{"line_number":468,"context_line":"        is_race_request \u003d False"}],"source_content_type":"text/x-python","patch_set":15,"id":"0a18ee09_a8f3791d","line":465,"updated":"2026-05-13 03:24:34.000000000","message":"I guess in a v2 world all of these tests could \"just\" mock `time.time` inline so that the code under test can call `Timestamp.now(version\u003d2)` naturally and still get non-colliding timestamps (which would have totally different results than these tests)\n\nbut I think we can keep these tests mostly as is b/c\n\n1) the tests are still be useful to demonstrate the scary behavior for v1 timestamps\n\n2) they may serve as a path to understanding how we can teach the consistency engine to eventually help dedect/uncover situation so we can take action to repairing v1 timestamp collisions towards consistency","commit_id":"cf1537b6af35ec9277e3d254f39afd38ae578070"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":462,"context_line":"        now \u003d Timestamp.now()"},{"line_number":463,"context_line":"        base_headers \u003d {"},{"line_number":464,"context_line":"            \u0027x-timestamp\u0027: now.internal,"},{"line_number":465,"context_line":"        }"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"        num_sent \u003d 0"},{"line_number":468,"context_line":"        is_race_request \u003d False"}],"source_content_type":"text/x-python","patch_set":15,"id":"f8767d05_6eae31dc","line":465,"in_reply_to":"0a18ee09_a8f3791d","updated":"2026-05-15 15:20:14.000000000","message":"I\u0027m not sure I want to maintain tests that deliberately set up legacy conditions just to show what happens if we hadn\u0027t fixed a bug, so I\u0027d expect this test to be updated to use v2 timestamps once we have v2 timestamps for PUTs. They can still be forced to be *identical* timestamps.\n\nI guess the tests would be documenting \"here\u0027s what can happen in those very unlikely cases when jitter isn\u0027t enough\".","commit_id":"cf1537b6af35ec9277e3d254f39afd38ae578070"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9d73661780e26223f132106b7b86ec2ed7d7874b","unresolved":true,"context_lines":[{"line_number":491,"context_line":"                    is_race_request \u003d True"},{"line_number":492,"context_line":"                    self.swift.upload_object("},{"line_number":493,"context_line":"                        BytesIO(contents), self.account, self.container_name,"},{"line_number":494,"context_line":"                        self.object_name, headers\u003dheaders)"},{"line_number":495,"context_line":"                    is_race_request \u003d False"},{"line_number":496,"context_line":"                num_sent +\u003d 1"},{"line_number":497,"context_line":"                return orig_end_of_object_data(*args, **kwargs)"}],"source_content_type":"text/x-python","patch_set":15,"id":"8c4bc8fc_e2ad7025","line":494,"updated":"2026-05-13 03:24:34.000000000","message":"yo dawg! I heard you like complicated request race conditions so I put a upload_object up in your original upload\u0027s end_of_object_data so you can upload while you upload!","commit_id":"cf1537b6af35ec9277e3d254f39afd38ae578070"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        return [df for df in data_files"},{"line_number":166,"context_line":"                if os.path.splitext(df)[0].endswith(\u0027#d\u0027)]"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def test_overlapping_commit(self):"},{"line_number":169,"context_line":"        contents \u003d b\u0027asdfb\u0027"},{"line_number":170,"context_line":"        now \u003d Timestamp.now()"},{"line_number":171,"context_line":"        orig_headers \u003d {"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f6b5e6e_82e8249e","line":168,"updated":"2026-05-15 15:20:14.000000000","message":"this scenario is ok","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":242,"context_line":"                self.fail(\u0027%s did not have foo metadata: %s\u0027 % (df, m))"},{"line_number":243,"context_line":"        self.assertEqual(counts, {\u0027foo\u0027: 6})"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"    def test_overlap_data_write_streams(self):"},{"line_number":246,"context_line":"        start_chr \u003d ord(\u0027a\u0027)"},{"line_number":247,"context_line":"        # assuming segments are 1MiB"},{"line_number":248,"context_line":"        num_segments \u003d 3"}],"source_content_type":"text/x-python","patch_set":16,"id":"7dffff7f_702fe7ad","line":245,"updated":"2026-05-15 15:20:14.000000000","message":"this scenario is not ok -\u003e inconsistent metadata","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":342,"context_line":"            {\u0027foo\u0027: 2, \u0027bar\u0027: 4},"},{"line_number":343,"context_line":"        ])"},{"line_number":344,"context_line":""},{"line_number":345,"context_line":"    def test_overlap_to_handoffs_collapses_to_unreadable(self):"},{"line_number":346,"context_line":"        old_contents \u003d b\u0027a\u0027 * 97"},{"line_number":347,"context_line":"        new_contents \u003d b\u0027b\u0027 * 97"},{"line_number":348,"context_line":"        now \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":16,"id":"0800893c_b74a9f5d","line":345,"updated":"2026-05-15 15:20:14.000000000","message":"this scenario is not ok?","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":425,"context_line":"            self.account, self.container_name,"},{"line_number":426,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"},{"line_number":427,"context_line":""},{"line_number":428,"context_line":"    def test_replicator_race(self):"},{"line_number":429,"context_line":"        # Tests a race condition where object replication runs concurrently"},{"line_number":430,"context_line":"        # with an in-progress replicated object upload. The test triggers"},{"line_number":431,"context_line":"        # the replicator after the first replica completes but while remaining"}],"source_content_type":"text/x-python","patch_set":16,"id":"1b48afd6_4a513c63","line":428,"updated":"2026-05-15 15:20:14.000000000","message":"this scenario is ok","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":479,"context_line":"                         self.policy.object_ring.replica_count)"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"        # Verify all replicas have the same timestamp"},{"line_number":482,"context_line":"        timestamps \u003d [resp[\u0027X-Timestamp\u0027] for resp in head_responses]"},{"line_number":483,"context_line":"        self.assertEqual(timestamps[0], str(now.normal))"},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"        # Verify content regardless of encryption turned on or off."}],"source_content_type":"text/x-python","patch_set":16,"id":"3cb93d5c_b70f0188","line":482,"updated":"2026-05-15 15:20:14.000000000","message":"I *think* we\u0027d get x-backend-timestamp back from an internal client, so that can be compared with ``now.internal`` - no need to approximate","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":480,"context_line":""},{"line_number":481,"context_line":"        # Verify all replicas have the same timestamp"},{"line_number":482,"context_line":"        timestamps \u003d [resp[\u0027X-Timestamp\u0027] for resp in head_responses]"},{"line_number":483,"context_line":"        self.assertEqual(timestamps[0], str(now.normal))"},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"        # Verify content regardless of encryption turned on or off."},{"line_number":486,"context_line":"        _, _, body_iter \u003d self.swift.get_object("}],"source_content_type":"text/x-python","patch_set":16,"id":"52562fae_993a48b6","line":483,"updated":"2026-05-15 15:20:14.000000000","message":"this isn\u0027t verifying all replicas have the same timestamp as the comment suggests?","commit_id":"eb645619550136db9623820775b579df55a82008"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":489,"context_line":"        self.assertEqual("},{"line_number":490,"context_line":"            body, contents, \"Content mismatch when reading through proxy\")"},{"line_number":491,"context_line":""},{"line_number":492,"context_line":"    def test_request_race(self):"},{"line_number":493,"context_line":"        # Tests a race condition where two concurrent PUT requests with"},{"line_number":494,"context_line":"        # identical timestamps but different metadatas (to simulate different"},{"line_number":495,"context_line":"        # ETags due to different IVs used) write to the same object."}],"source_content_type":"text/x-python","patch_set":16,"id":"b16531cd_cb192f51","line":492,"updated":"2026-05-15 15:20:14.000000000","message":"I think it would be useful to very clearly distinguish tests that verify something is ok vs those that are revealing a scenario that does not work. So for example:\n\n``test_request_race_leaves_inconsistent_metadata``","commit_id":"eb645619550136db9623820775b579df55a82008"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":1889,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":1890,"context_line":"        self.assertEqual(resp.status_int, 412)"},{"line_number":1891,"context_line":""},{"line_number":1892,"context_line":"    def test_PUT_timestamp_collision(self):"},{"line_number":1893,"context_line":"        t0 \u003d self.ts()"},{"line_number":1894,"context_line":"        req \u003d Request.blank("},{"line_number":1895,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"}],"source_content_type":"text/x-python","patch_set":18,"id":"d01fbfaf_895c832d","line":1892,"updated":"2026-05-15 15:20:14.000000000","message":"nit: could this be named ``test_PUT_timestamp_collision_no_race``\n\nmade me realise that our use of \"collision\" as a *bad thing* has always assumed some degree of concurrency","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":1918,"context_line":"        self.assertTrue(line.endswith(suffix), line)"},{"line_number":1919,"context_line":"        metadata_comparison \u003d line[len(prefix):-len(suffix)]"},{"line_number":1920,"context_line":"        decoder \u003d json.JSONDecoder()"},{"line_number":1921,"context_line":"        # raw_decode will consume one json object and stop:"},{"line_number":1922,"context_line":"        # https://docs.python.org/3/library/json.html#json.JSONDecoder.raw_decode"},{"line_number":1923,"context_line":"        emeta, end \u003d decoder.raw_decode(metadata_comparison)"},{"line_number":1924,"context_line":"        op, metadata_comparison \u003d metadata_comparison[end:].split(None, 1)"}],"source_content_type":"text/x-python","patch_set":18,"id":"90beb746_293adef3","line":1921,"updated":"2026-05-15 15:20:14.000000000","message":"TIL, nice","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a9b8fd54dedc735b04a493a4bdf8ccef3b15a53b","unresolved":false,"context_lines":[{"line_number":1918,"context_line":"        self.assertTrue(line.endswith(suffix), line)"},{"line_number":1919,"context_line":"        metadata_comparison \u003d line[len(prefix):-len(suffix)]"},{"line_number":1920,"context_line":"        decoder \u003d json.JSONDecoder()"},{"line_number":1921,"context_line":"        # raw_decode will consume one json object and stop:"},{"line_number":1922,"context_line":"        # https://docs.python.org/3/library/json.html#json.JSONDecoder.raw_decode"},{"line_number":1923,"context_line":"        emeta, end \u003d decoder.raw_decode(metadata_comparison)"},{"line_number":1924,"context_line":"        op, metadata_comparison \u003d metadata_comparison[end:].split(None, 1)"}],"source_content_type":"text/x-python","patch_set":18,"id":"63210fee_104f2196","line":1921,"in_reply_to":"90beb746_293adef3","updated":"2026-06-01 09:55:11.000000000","message":"Acknowledged","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":1974,"context_line":"            self.testdir, \u0027sda1\u0027,"},{"line_number":1975,"context_line":"            storage_directory(\u0027objects\u0027, \u0027p\u0027, hash_path(\u0027a\u0027, \u0027c\u0027, \u0027o\u0027)),"},{"line_number":1976,"context_line":"            \u0027%s.data\u0027 % t0.internal)"},{"line_number":1977,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":1978,"context_line":""},{"line_number":1979,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":1980,"context_line":"    def test_PUT_timestamp_collision_linkat_race_filtered_metadata_match(self):"}],"source_content_type":"text/x-python","patch_set":18,"id":"013db29b_3d205772","line":1977,"updated":"2026-05-15 15:20:14.000000000","message":"we could/should assert expected on disk metadata, see https://review.opendev.org/c/openstack/swift/+/988781","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a9b8fd54dedc735b04a493a4bdf8ccef3b15a53b","unresolved":false,"context_lines":[{"line_number":1974,"context_line":"            self.testdir, \u0027sda1\u0027,"},{"line_number":1975,"context_line":"            storage_directory(\u0027objects\u0027, \u0027p\u0027, hash_path(\u0027a\u0027, \u0027c\u0027, \u0027o\u0027)),"},{"line_number":1976,"context_line":"            \u0027%s.data\u0027 % t0.internal)"},{"line_number":1977,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":1978,"context_line":""},{"line_number":1979,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":1980,"context_line":"    def test_PUT_timestamp_collision_linkat_race_filtered_metadata_match(self):"}],"source_content_type":"text/x-python","patch_set":18,"id":"a330eda0_97af3132","line":1977,"in_reply_to":"013db29b_3d205772","updated":"2026-06-01 09:55:11.000000000","message":"Done","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":1977,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":1978,"context_line":""},{"line_number":1979,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":1980,"context_line":"    def test_PUT_timestamp_collision_linkat_race_filtered_metadata_match(self):"},{"line_number":1981,"context_line":"        t0 \u003d self.ts()"},{"line_number":1982,"context_line":"        etag \u003d \u00270b4c12d7e0a73840c1c4f148fda3b037\u0027"},{"line_number":1983,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":18,"id":"e06bfce7_d46d8cd3","line":1980,"updated":"2026-05-15 15:20:14.000000000","message":"nit: add \n``# but not all metadata matches``","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a9b8fd54dedc735b04a493a4bdf8ccef3b15a53b","unresolved":false,"context_lines":[{"line_number":1977,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":1978,"context_line":""},{"line_number":1979,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":1980,"context_line":"    def test_PUT_timestamp_collision_linkat_race_filtered_metadata_match(self):"},{"line_number":1981,"context_line":"        t0 \u003d self.ts()"},{"line_number":1982,"context_line":"        etag \u003d \u00270b4c12d7e0a73840c1c4f148fda3b037\u0027"},{"line_number":1983,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":18,"id":"c44ecc23_3ea84588","line":1980,"in_reply_to":"e06bfce7_d46d8cd3","updated":"2026-06-01 09:55:11.000000000","message":"Done","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2b035892e4a4b956e84613d3b97a72f2f21cc7a4","unresolved":true,"context_lines":[{"line_number":2028,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":2029,"context_line":""},{"line_number":2030,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":2031,"context_line":"    def test_PUT_timestamp_collision_linkat_race_metadata_match(self):"},{"line_number":2032,"context_line":"        t0 \u003d self.ts()"},{"line_number":2033,"context_line":"        req \u003d Request.blank("},{"line_number":2034,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"}],"source_content_type":"text/x-python","patch_set":18,"id":"4ab146ae_2b42a1fb","line":2031,"updated":"2026-05-15 15:20:14.000000000","message":"nit: could this be renamed\n``test_PUT_timestamp_collision_linkat_race_all_metadata_match``\n\n...I got a little confused about what was (not) matching in this and the previous test","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a9b8fd54dedc735b04a493a4bdf8ccef3b15a53b","unresolved":false,"context_lines":[{"line_number":2028,"context_line":"        self.assertEqual([expected_file_path], calls)"},{"line_number":2029,"context_line":""},{"line_number":2030,"context_line":"    @requires_o_tmpfile_support_in_tmp"},{"line_number":2031,"context_line":"    def test_PUT_timestamp_collision_linkat_race_metadata_match(self):"},{"line_number":2032,"context_line":"        t0 \u003d self.ts()"},{"line_number":2033,"context_line":"        req \u003d Request.blank("},{"line_number":2034,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"}],"source_content_type":"text/x-python","patch_set":18,"id":"b550ce57_f8b1b227","line":2031,"in_reply_to":"4ab146ae_2b42a1fb","updated":"2026-06-01 09:55:11.000000000","message":"Done","commit_id":"ada6585714967f3cfd0f78248ca8860a756b583d"}]}