)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This is totally not necessary, as those stale delete tasks could be"},{"line_number":19,"context_line":"deleted as long as their creation timestamps were older than the"},{"line_number":20,"context_line":"timestamp of the new \u0027x-delete-at\u0027 POST."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: I07a82025658eeabf462b8e26b7bc352530ba88b9"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"04d013b1_9f36599e","line":20,"updated":"2024-12-14 00:32:10.000000000","message":"that\u0027s exactly right, good explination!","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This is totally not necessary, as those stale delete tasks could be"},{"line_number":19,"context_line":"deleted as long as their creation timestamps were older than the"},{"line_number":20,"context_line":"timestamp of the new \u0027x-delete-at\u0027 POST."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: I07a82025658eeabf462b8e26b7bc352530ba88b9"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"eb86f2f9_8da65a70","line":20,"in_reply_to":"04d013b1_9f36599e","updated":"2025-01-21 16:50:52.000000000","message":"Acknowledged","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"57792449e7e58db0c3deff89d1a97e5a15533be5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"50e55e53_9348e32b","updated":"2024-10-21 14:28:06.000000000","message":"I\u0027d like to see a probe test that some how forces the clean-up async to not get processed.  I think if you modify and objects x-delete-at (or remove it) and don\u0027t run the object-updater you should be able to force an expirer to hit the 412\n\nI\u0027m not sure yet that\u0027s it\u0027s going to be possible to make the determination of clean-up in the expirer by returning extra metadata from ALL the object-serverS - I think we need each object server to look at the x-if-delete-at requests\u0027 last-modified time (x-expirer-queue-timestamp or something) and if decide if their metadata is newer than the queue entry.  A quorum of 409s should already cause the expirer to pop the queue.","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"93675d47a103e3444f2ef3f1e38966d8855d9069","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"6fbe0b6d_80c7403f","in_reply_to":"50e55e53_9348e32b","updated":"2024-10-22 18:46:47.000000000","message":"\u003e I\u0027d like to see a probe test that some how forces the clean-up async to not get processed.\n\nhttps://review.opendev.org/c/openstack/swift/+/803406 might offer a decent starting point -- though I think you\u0027ll want to POST to one half, sleep until it\u0027s active, then POST to the other half.","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"00b88af5_851d8997","updated":"2024-12-14 00:32:10.000000000","message":"I think I convinced myself that this is a good idea and would work as-is, but it took a lot of work.\n\n937745: sq? moar tests for expirer-409 | https://review.opendev.org/c/openstack/swift/+/937745\n\nMaybe at a minimum we should consider dragging in the probe-test and making sure it passes/fails as expected with this change?\n\nthe only other blocker might be `X-Backend-Expirer-Task-Timestamp`; I haven\u0027t tested it, but now I\u0027m wondering if a client could make a DELETE request with X-If-Delete-At.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c787f9d074b3b3ab8421d8cc701aeeccedea6ff","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"152094a4_99abe7b8","updated":"2024-11-27 17:23:22.000000000","message":"recheck\n\nfunc-cors test has now been fixed","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"2a1dea57cba79ccf65a8ce1df7598d8dd9c385e8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"2950b880_e1e09b99","updated":"2024-11-20 06:01:57.000000000","message":"still working on the probe test","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6db377d65e1c47cb6fce345d26847e9edd5155b3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"d01706be_e4fabf4e","in_reply_to":"00b88af5_851d8997","updated":"2025-01-15 18:45:37.000000000","message":"\u003e Maybe at a minimum we should consider dragging in the probe-test and making sure it passes/fails as expected with this change?\n\nThe probe test I linked is just investigating/demonstrating https://bugs.launchpad.net/swift/+bug/1182628 -- we\u0027ll need to write a new one if we want to demonstrate what\u0027s fixed here. `X-If-Delete-At` will still be just as broken with regard to #1182628\n\n\u003e now I\u0027m wondering if a client could make a DELETE request with X-If-Delete-At.\n\nYes, they can:\n```\nvagrant@saio:/vagrant/requirements$ swift upload c tox.ini \ntox.ini\nvagrant@saio:/vagrant/requirements$ swift delete c tox.ini -H x-if-delete-at:123456789\nError Deleting: c/tox.ini: Object DELETE failed: http://saio:8080/v1/AUTH_test/c/tox.ini 412 Precondition Failed   b\u0027X-If-Delete-At and X-Delete-At do not match\u0027 (txn: tx751626ac8a6f41eabe43e-006787fe8d)\n```\n\nI like the idea of making the new timestamp a backend header; `X-If-Delete-At` probably should have been the whole time.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"960a50cbe02e73ee1ef0ff5158cab20d8f6111d9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"b287b828_5878eaf7","in_reply_to":"d01706be_e4fabf4e","updated":"2025-01-16 19:39:50.000000000","message":"\u003e we\u0027ll need to write a new one if we want to demonstrate what\u0027s fixed here\n\ni.e. 937745: sq? moar tests for expirer-409 | https://review.opendev.org/c/openstack/swift/+/937745\n\nwith *only* obj/expirer.py reverted:\n\n```\n        # object is now reaped\n        self.assertObjectNotInListing()\n        # all queue tasks are popped\n\u003e       self.assertFalse(self.gather_expirer_tasks())\nE       AssertionError: [{\u0027bytes\u0027: 0, \u0027hash\u0027: \u0027d41d8cd98f00b204e9800998ecf8427e\u0027, \u0027name\u0027: \u00271737055782-AUTH_test/container-13db3b97-e9d3-423d-9545-f597d5c88ef7/object-d400f71c-d720-493c-9ff2-e83d2ae04b8a\u0027, \u0027content_type\u0027: \u0027text/plain;swift_expirer_bytes\u003d0\u0027, \u0027last_modified\u0027: \u00272025-01-16T19:29:39.175950\u0027}] is not false\n```\n\nwith *only* obj/server.py reverted:\n\n```\n        # object is now reaped\n        self.assertObjectNotInListing()\n        # all queue tasks are popped\n\u003e       self.assertFalse(self.gather_expirer_tasks())\nE       AssertionError: [{\u0027bytes\u0027: 0, \u0027hash\u0027: \u0027d41d8cd98f00b204e9800998ecf8427e\u0027, \u0027name\u0027: \u00271737055962-AUTH_test/container-698bc0eb-5430-4566-97a4-6e789ffe8f9f/object-17da63a7-640f-467e-b9b2-75ae70e45f20\u0027, \u0027content_type\u0027: \u0027text/plain;swift_expirer_bytes\u003d0\u0027, \u0027last_modified\u0027: \u00272025-01-16T19:32:39.596520\u0027}] is not false\n```\n\ni.e. this change is an early queue pop when x-delete-at get\u0027s updated (newer last_modified/x-delete-task-timestamp) and during upgrade it\u0027s \"the same as current behavior: retry until a reclaim age)","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"960a50cbe02e73ee1ef0ff5158cab20d8f6111d9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"7d6125f0_72bb24b6","updated":"2025-01-16 19:39:50.000000000","message":"I think most of my comments can be ack\u0027d if we squash my follow-up\n\n937745: sq? moar tests for expirer-409 | https://review.opendev.org/c/openstack/swift/+/937745\n\nIt looks like tim had a suggestion to improve robustness in the object-server path, I think I had some unaddressed comments there as well - BUT since I think with all the work on tests I\u0027ve convinced myself it\u0027s correct *as written* any re-wording would be non-material changes and shouldn\u0027t block merge.\n\nThe one thing we should probably do before merge is s/X-Delete-Task-Timestamp/X-Backend-Delete-Task-Timestamp/g\n\n... but I think we could carry this patch as-in if we want to see our expirer stats graphs wiggle on 409 vs 412.","commit_id":"7dae5df3afe3f5bad9b05a699b24f2667618b2d9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"bcd596a4_063313d5","updated":"2025-01-22 17:02:02.000000000","message":"I\u0027m still ok to carry this... maybe it should go ahead and merge - I\u0027m just writing more tests:\n\n939828: sq: add test for parse_error | https://review.opendev.org/c/openstack/swift/+/939828\n\n... and at least one of those is just exploring a different/related behavior of \"stale\" tasks that encounter tombstones as opposed to [meta]data - and those are still returning 412 (so no worse)","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"466f456ed8fb483342d4f5cccb92e43dfd74205d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"32c1d42d_a901be4f","updated":"2025-01-31 20:24:33.000000000","message":"this is working; it\u0027s complete, correct and could merge.\n\nthe parse_error tests in the sq might be helpful - but almost any change could benefit from \"more testing\"\n\nthe \"messiness\" of the double 409 handling in the object-server was pre-eixisting. I don\u0027t have the bandwidth to fix it now so I wouldn\u0027t expect the author to.\n\nI\u0027m not sure I believe the stale-task-over-newer-tombstone-should-409 can be implemented reliably; and the current 412/404 response (depending on if the tombstones\u0027 been reclaimed?) is not *incorrect* and could be optimized later if it\u0027s an issue.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"8f3b1656_b7a37fdf","updated":"2025-02-04 21:51:11.000000000","message":"Fixed a test case when object-server can\u0027t parse the new added backend timestamp header value.\n\nAlso add Tim into co-author, since he came up with the original idea on this fix.","commit_id":"b1f3742419a5e4e8cf91cbe47d6f67b743a7df9a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"bc50ace3_58517f55","updated":"2025-02-11 03:52:45.000000000","message":"Looking great, just some nits really. And a bit of readiblity. Nothing major. Looks good.\n\nI wonder when we plan to land this chain ;)","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"},{"author":{"_account_id":35790,"name":"Shreeya Deshpande","email":"shreeyad@nvidia.com","username":"shreeyad"},"change_message_id":"59e851b888394c1df4a2997a5f9eeea5c6f3c8c3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":30,"id":"b61f6456_f0688848","updated":"2026-03-03 16:18:01.000000000","message":"recheck","commit_id":"842de095b24deb5a207961ae53313187553ca49a"}],"swift/obj/expirer.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"57792449e7e58db0c3deff89d1a97e5a15533be5","unresolved":true,"context_lines":[{"line_number":892,"context_line":"        else:"},{"line_number":893,"context_line":"            # response includes latest x-delete-at timestamp from object-server"},{"line_number":894,"context_line":"            if resp_x_delete_at \u003e last_modified:"},{"line_number":895,"context_line":"                return True"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"        return False"},{"line_number":898,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"20940160_e1d10c6f","line":895,"updated":"2024-10-21 14:28:06.000000000","message":"I don\u0027t think this is the correct evaluation.\n\nMost *likely* if the current x-delete-at on the object doesn\u0027t match it\u0027s becase the user \"extended\" the expiration of the object.  But there\u0027s also a reasonable scenario where the user did a POST with \"x-remove-delete-at\"\n\n```\nvagrant@saio:~$ swift stat test test\n               Account: AUTH_test\n             Container: test\n                Object: test\n          Content Type: application/octet-stream\n        Content Length: 8\n         Last Modified: Mon, 21 Oct 2024 14:21:25 GMT\n                  ETag: \"70c1db56f301c9e337b0099bd4174b28\"\n            Meta Mtime: 1725482069.227783\n           X-Delete-At: 1729520514\n           X-Timestamp: 1729520484.25558\n         Accept-Ranges: bytes\n            X-Trans-Id: txe0ffae465b1c4de8b1c73-0067166369\nX-Openstack-Request-Id: txe0ffae465b1c4de8b1c73-0067166369\nvagrant@saio:~$ swift post test test -H \u0027x-remove-delete-at: true\u0027\nvagrant@saio:~$ swift stat test test\n               Account: AUTH_test\n             Container: test\n                Object: test\n          Content Type: application/octet-stream\n        Content Length: 8\n         Last Modified: Mon, 21 Oct 2024 14:21:40 GMT\n                  ETag: \"70c1db56f301c9e337b0099bd4174b28\"\n           X-Timestamp: 1729520499.65097\n         Accept-Ranges: bytes\n            X-Trans-Id: tx9284ade45c8444449a1d2-0067166378\nX-Openstack-Request-Id: tx9284ade45c8444449a1d2-0067166378\n```\n\nIf the old queue entry didn\u0027t get cleaned-up it\u0027s going to be stale, but not because the x-delete-at *value* on the object is larger than the queue entry.  But because the object\u0027s metadata timestamp is newer than the last_modified of the queue entry!","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":false,"context_lines":[{"line_number":892,"context_line":"        else:"},{"line_number":893,"context_line":"            # response includes latest x-delete-at timestamp from object-server"},{"line_number":894,"context_line":"            if resp_x_delete_at \u003e last_modified:"},{"line_number":895,"context_line":"                return True"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"        return False"},{"line_number":898,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"db65f80a_5dcc20e8","line":895,"in_reply_to":"20940160_e1d10c6f","updated":"2024-12-14 00:32:10.000000000","message":"Done","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"57792449e7e58db0c3deff89d1a97e5a15533be5","unresolved":true,"context_lines":[{"line_number":993,"context_line":"        a, c, o \u003d split_path(\u0027/\u0027 + actual_obj, 3, 3, True)"},{"line_number":994,"context_line":"        resp \u003d self.swift.delete_object("},{"line_number":995,"context_line":"            a, c, o,"},{"line_number":996,"context_line":"            headers\u003dheaders, acceptable_statuses\u003dacceptable_statuses)"},{"line_number":997,"context_line":"        upload_id_key \u003d s3_sysmeta_header(\u0027object\u0027, \u0027upload-id\u0027)"},{"line_number":998,"context_line":"        segments_key \u003d s3_sysmeta_header(\u0027object\u0027, \u0027etag\u0027)"},{"line_number":999,"context_line":"        if not (is_success(resp.status_int) and all("}],"source_content_type":"text/x-python","patch_set":2,"id":"dfb596a2_3749ea7e","line":996,"updated":"2024-10-21 14:28:06.000000000","message":"since we\u0027re using InternalClient there\u0027s no way for the response to include all the metadata timestamp from all the replicas.\n\nRealistically if ANY replica has an x-delete-value who\u0027s metadata timestamp indicates it was modified after the current task-queue-entry was created (either extending OR removing the actual x-delete-at value) then we can probably pop THIS queue entry.","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":false,"context_lines":[{"line_number":993,"context_line":"        a, c, o \u003d split_path(\u0027/\u0027 + actual_obj, 3, 3, True)"},{"line_number":994,"context_line":"        resp \u003d self.swift.delete_object("},{"line_number":995,"context_line":"            a, c, o,"},{"line_number":996,"context_line":"            headers\u003dheaders, acceptable_statuses\u003dacceptable_statuses)"},{"line_number":997,"context_line":"        upload_id_key \u003d s3_sysmeta_header(\u0027object\u0027, \u0027upload-id\u0027)"},{"line_number":998,"context_line":"        segments_key \u003d s3_sysmeta_header(\u0027object\u0027, \u0027etag\u0027)"},{"line_number":999,"context_line":"        if not (is_success(resp.status_int) and all("}],"source_content_type":"text/x-python","patch_set":2,"id":"c3d6c863_6c50790b","line":996,"in_reply_to":"dfb596a2_3749ea7e","updated":"2024-12-14 00:32:10.000000000","message":"Acknowledged","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":652,"context_line":"                                         acceptable_statuses\u003d[2]):"},{"line_number":653,"context_line":"            container_empty \u003d False"},{"line_number":654,"context_line":"            task_object \u003d o[\u0027name\u0027].encode(\u0027utf8\u0027) if six.PY2 else o[\u0027name\u0027]"},{"line_number":655,"context_line":"            last_modified_str \u003d o.get(\u0027last_modified\u0027)"},{"line_number":656,"context_line":"            last_modified \u003d last_modified_date_to_timestamp("},{"line_number":657,"context_line":"                last_modified_str) if last_modified_str else None"},{"line_number":658,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"25cecedf_36656750","line":655,"updated":"2024-12-14 00:32:10.000000000","message":"can this ever *really* be missing from the listing or is this more of a test thing?\n\nLike if the `iter_objects` call used a `delimiter\u003d` I think we could potentially return \"subdir\" entries that don\u0027t have a `last_modified` key but I don\u0027t think this should be needed...","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":652,"context_line":"                                         acceptable_statuses\u003d[2]):"},{"line_number":653,"context_line":"            container_empty \u003d False"},{"line_number":654,"context_line":"            task_object \u003d o[\u0027name\u0027].encode(\u0027utf8\u0027) if six.PY2 else o[\u0027name\u0027]"},{"line_number":655,"context_line":"            last_modified_str \u003d o.get(\u0027last_modified\u0027)"},{"line_number":656,"context_line":"            last_modified \u003d last_modified_date_to_timestamp("},{"line_number":657,"context_line":"                last_modified_str) if last_modified_str else None"},{"line_number":658,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"2d1f13c2_4c8f2393","line":655,"in_reply_to":"25cecedf_36656750","updated":"2025-01-21 16:50:52.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":663,"context_line":"                                      task_object)"},{"line_number":664,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"},{"line_number":665,"context_line":"                continue"},{"line_number":666,"context_line":"            is_async \u003d o.get(\u0027content_type\u0027) \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":667,"context_line":"            delay_reaping \u003d self.get_delay_reaping(target_account,"},{"line_number":668,"context_line":"                                                   target_container)"},{"line_number":669,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"22ca52c8_8a38773b","line":666,"updated":"2024-12-14 00:32:10.000000000","message":"ok, this is cargo-culting the idea that our test/listing stubs don\u0027t have to have realistic looking container listings - I think we have a an oppertunity to fix that now.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":663,"context_line":"                                      task_object)"},{"line_number":664,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"},{"line_number":665,"context_line":"                continue"},{"line_number":666,"context_line":"            is_async \u003d o.get(\u0027content_type\u0027) \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":667,"context_line":"            delay_reaping \u003d self.get_delay_reaping(target_account,"},{"line_number":668,"context_line":"                                                   target_container)"},{"line_number":669,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"a7fe3eec_c16353e5","line":666,"in_reply_to":"22ca52c8_8a38773b","updated":"2025-01-21 16:50:52.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":692,"context_line":"                       target_account, target_container, target_object]),"},{"line_number":693,"context_line":"                   \u0027delete_timestamp\u0027: delete_timestamp,"},{"line_number":694,"context_line":"                   \u0027is_async_delete\u0027: is_async,"},{"line_number":695,"context_line":"                   \u0027task_timestamp\u0027: last_modified}"},{"line_number":696,"context_line":"        if container_empty:"},{"line_number":697,"context_line":"            try:"},{"line_number":698,"context_line":"                self.swift.delete_container("}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa3da85_42b47015","line":695,"updated":"2024-12-14 00:32:10.000000000","message":"ok, so this is a new key in every task_obj dict that comes out of the iter.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":692,"context_line":"                       target_account, target_container, target_object]),"},{"line_number":693,"context_line":"                   \u0027delete_timestamp\u0027: delete_timestamp,"},{"line_number":694,"context_line":"                   \u0027is_async_delete\u0027: is_async,"},{"line_number":695,"context_line":"                   \u0027task_timestamp\u0027: last_modified}"},{"line_number":696,"context_line":"        if container_empty:"},{"line_number":697,"context_line":"            try:"},{"line_number":698,"context_line":"                self.swift.delete_container("}],"source_content_type":"text/x-python","patch_set":3,"id":"127268d8_e4b8a130","line":695,"in_reply_to":"3fa3da85_42b47015","updated":"2025-01-21 16:50:52.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":857,"context_line":""},{"line_number":858,"context_line":"    def delete_object(self, target_path, delete_timestamp,"},{"line_number":859,"context_line":"                      task_account, task_container, task_object,"},{"line_number":860,"context_line":"                      is_async_delete, task_timestamp\u003dNone):"},{"line_number":861,"context_line":"        start_time \u003d time()"},{"line_number":862,"context_line":"        try:"},{"line_number":863,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"84dcfbab_1e4a5d15","line":860,"updated":"2024-12-14 00:32:10.000000000","message":"when do we ever call this method without this task_timestamp \"kwarg\" - it seems like it\u0027s just in tests; and therefore, those tests should be updated.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":857,"context_line":""},{"line_number":858,"context_line":"    def delete_object(self, target_path, delete_timestamp,"},{"line_number":859,"context_line":"                      task_account, task_container, task_object,"},{"line_number":860,"context_line":"                      is_async_delete, task_timestamp\u003dNone):"},{"line_number":861,"context_line":"        start_time \u003d time()"},{"line_number":862,"context_line":"        try:"},{"line_number":863,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"a22438a6_fcaeb153","line":860,"in_reply_to":"84dcfbab_1e4a5d15","updated":"2025-01-21 16:50:52.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":918,"context_line":"                self.logger.increment(\u0027segments\u0027)"},{"line_number":919,"context_line":""},{"line_number":920,"context_line":"    def delete_actual_object(self, actual_obj, timestamp,"},{"line_number":921,"context_line":"                             is_async_delete, task_timestamp\u003dNone):"},{"line_number":922,"context_line":"        \"\"\""},{"line_number":923,"context_line":"        Deletes the end-user object indicated by the actual object name given"},{"line_number":924,"context_line":"        \u0027\u003caccount\u003e/\u003ccontainer\u003e/\u003cobject\u003e\u0027."}],"source_content_type":"text/x-python","patch_set":3,"id":"612968d8_8827b920","line":921,"updated":"2024-12-14 00:32:10.000000000","message":"`is_async_delete` is a good example of this method growing a param and tests adjusting to call this method with the new/correct param.\n\n`task_timestamp\u003dNone` doesn\u0027t even make sense, just as `is_async_delete\u003dFalse` may have *seemed* like a reasonable \"default\" it\u0027s not needed; a task MUST have a content_type and MUST be classified as async or not.  Similarly how can a task NOT have a `last_modified` and there for it must have a task_timestamp.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"95b8299cd722170eb665e3087e16e025a793de02","unresolved":false,"context_lines":[{"line_number":918,"context_line":"                self.logger.increment(\u0027segments\u0027)"},{"line_number":919,"context_line":""},{"line_number":920,"context_line":"    def delete_actual_object(self, actual_obj, timestamp,"},{"line_number":921,"context_line":"                             is_async_delete, task_timestamp\u003dNone):"},{"line_number":922,"context_line":"        \"\"\""},{"line_number":923,"context_line":"        Deletes the end-user object indicated by the actual object name given"},{"line_number":924,"context_line":"        \u0027\u003caccount\u003e/\u003ccontainer\u003e/\u003cobject\u003e\u0027."}],"source_content_type":"text/x-python","patch_set":3,"id":"8e39fc4b_c1fdb877","line":921,"in_reply_to":"612968d8_8827b920","updated":"2025-01-21 16:50:52.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":948,"context_line":"                       \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":949,"context_line":"                       \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027}"},{"line_number":950,"context_line":"            if task_timestamp:"},{"line_number":951,"context_line":"                headers[\u0027X-Delete-Task-Timestamp\u0027] \u003d task_timestamp.normal"},{"line_number":952,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":953,"context_line":"        a, c, o \u003d split_path(\u0027/\u0027 + actual_obj, 3, 3, True)"},{"line_number":954,"context_line":"        resp \u003d self.swift.delete_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"001840e7_bf903f58","line":951,"updated":"2024-12-14 00:32:10.000000000","message":"ts.normal vs ts.internal doesn\u0027t matter here since we\u0027re extracting from a last_modified the offset will always be zero - but it\u0027s probably better to use `.internal` when reasonable.\n\nThis might look better as a `X-Backend-` header since we never expect clients to send it.\n\nThere\u0027s no clear precedence and it doesn\u0027t make it any easier to change across upgrades; but for explicitness might be better as:\n\n```\nX_TASK_TIMESTAMP \u003d \u0027X-Backend-Expirer-Task-Timestamp\u0027\n```\n\nsince both the obj.server and obj.expirer need to use the same header key.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ed376503266c2adca61fa60c5680c735c1660b45","unresolved":false,"context_lines":[{"line_number":948,"context_line":"                       \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":949,"context_line":"                       \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027}"},{"line_number":950,"context_line":"            if task_timestamp:"},{"line_number":951,"context_line":"                headers[\u0027X-Delete-Task-Timestamp\u0027] \u003d task_timestamp.normal"},{"line_number":952,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":953,"context_line":"        a, c, o \u003d split_path(\u0027/\u0027 + actual_obj, 3, 3, True)"},{"line_number":954,"context_line":"        resp \u003d self.swift.delete_object("}],"source_content_type":"text/x-python","patch_set":3,"id":"f72e79db_ca934957","line":951,"in_reply_to":"001840e7_bf903f58","updated":"2025-01-21 18:29:11.000000000","message":"``X-Backend-`` is better, I have changed the header name to \u0027X-Backend-Expirer-Task-Timestamp\u0027.\n\nI searched the usages for other backend headers, even though they are used for multiple places, the current usage pattern is to just use the name instead of defining a global variable. For example, for the backend header ``\u0027X-Backend-Clean-Expiring-Object-Queue\u0027``, it\u0027s also used in obj.py, server.py and expirer.py, but its full name are used in those places. I will follow the same pattern.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":656,"context_line":"            try:"},{"line_number":657,"context_line":"                delete_timestamp, target_account, target_container, \\"},{"line_number":658,"context_line":"                    target_object \u003d parse_task_obj(task_object)"},{"line_number":659,"context_line":"            except ValueError:"},{"line_number":660,"context_line":"                self.logger.exception(\u0027Unexcepted error handling task %r\u0027 %"},{"line_number":661,"context_line":"                                      task_object)"},{"line_number":662,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"0c84d465_a0630f95","line":659,"updated":"2025-01-22 17:02:02.000000000","message":"FWIW this one seems to be tested!\n\n```\ntest-expirer ERROR: Exception while listing objects in container .expiring_objects 1737331126: name \u0027asdf\u0027 is not defined\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d short test summary info \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\nFAILED swift/test/unit/obj/test_expirer.py::TestObjectExpirer::test_iter_task_to_expire - AssertionError: Lists differ: [] !\u003d [{\u0027task_account\u0027: \u0027.expiring_objects\u0027, \u0027ta[1228 chars]00\u0027}]\n```","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":656,"context_line":"            try:"},{"line_number":657,"context_line":"                delete_timestamp, target_account, target_container, \\"},{"line_number":658,"context_line":"                    target_object \u003d parse_task_obj(task_object)"},{"line_number":659,"context_line":"            except ValueError:"},{"line_number":660,"context_line":"                self.logger.exception(\u0027Unexcepted error handling task %r\u0027 %"},{"line_number":661,"context_line":"                                      task_object)"},{"line_number":662,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9058eeae_171622a2","line":659,"in_reply_to":"0c84d465_a0630f95","updated":"2025-02-04 17:51:20.000000000","message":"Acknowledged","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":664,"context_line":"            try:"},{"line_number":665,"context_line":"                is_async \u003d o[\u0027content_type\u0027] \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":666,"context_line":"                last_modified_str \u003d o[\u0027last_modified\u0027]"},{"line_number":667,"context_line":"            except KeyError:"},{"line_number":668,"context_line":"                self.logger.exception(\u0027Unexcepted error handling task %r\u0027 %"},{"line_number":669,"context_line":"                                      task_object)"},{"line_number":670,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"d361c014_a1a73fb5","line":667,"updated":"2025-01-22 17:02:02.000000000","message":"unfortunately if I add a NameError under this block no unittests in `obj/test_expirer.py` seem to fail.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":false,"context_lines":[{"line_number":664,"context_line":"            try:"},{"line_number":665,"context_line":"                is_async \u003d o[\u0027content_type\u0027] \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":666,"context_line":"                last_modified_str \u003d o[\u0027last_modified\u0027]"},{"line_number":667,"context_line":"            except KeyError:"},{"line_number":668,"context_line":"                self.logger.exception(\u0027Unexcepted error handling task %r\u0027 %"},{"line_number":669,"context_line":"                                      task_object)"},{"line_number":670,"context_line":"                self.logger.increment(\u0027tasks.parse_errors\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"7a5cebc3_5b1a0ca6","line":667,"in_reply_to":"d361c014_a1a73fb5","updated":"2025-02-04 21:51:11.000000000","message":"Done","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"}],"swift/obj/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"57792449e7e58db0c3deff89d1a97e5a15533be5","unresolved":true,"context_lines":[{"line_number":1256,"context_line":"            else:"},{"line_number":1257,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1258,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1259,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1260,"context_line":"        try:"},{"line_number":1261,"context_line":"            req_if_delete_at_val \u003d request.headers[\u0027x-if-delete-at\u0027]"},{"line_number":1262,"context_line":"            req_if_delete_at \u003d Timestamp(req_if_delete_at_val)"}],"source_content_type":"text/x-python","patch_set":2,"id":"8b2a7155_1f02578c","line":1259,"updated":"2024-10-21 14:28:06.000000000","message":"if the current object doesn\u0027t HAVE an x-delete-at we set the value to zero, that\u0027s going to be OLDER than the queue entries timestamp (but it might be \"more correct\" if the object\u0027s metadata timestamp is larger than the last_modified we parsed from the queue entry)","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"91057478d097c30f459c30200b9e7113c7896696","unresolved":false,"context_lines":[{"line_number":1256,"context_line":"            else:"},{"line_number":1257,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1258,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1259,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1260,"context_line":"        try:"},{"line_number":1261,"context_line":"            req_if_delete_at_val \u003d request.headers[\u0027x-if-delete-at\u0027]"},{"line_number":1262,"context_line":"            req_if_delete_at \u003d Timestamp(req_if_delete_at_val)"}],"source_content_type":"text/x-python","patch_set":2,"id":"f0d381fb_5469ed79","line":1259,"in_reply_to":"8b2a7155_1f02578c","updated":"2024-11-20 05:59:26.000000000","message":"Acknowledged","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"57792449e7e58db0c3deff89d1a97e5a15533be5","unresolved":true,"context_lines":[{"line_number":1279,"context_line":"                response \u003d HTTPPreconditionFailed("},{"line_number":1280,"context_line":"                    request\u003drequest,"},{"line_number":1281,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1282,"context_line":"                response.headers[\u0027X-Delete-At\u0027] \u003d orig_delete_at.normal"},{"line_number":1283,"context_line":"                return response"},{"line_number":1284,"context_line":"            else:"},{"line_number":1285,"context_line":"                # differentiate success from no object at all"}],"source_content_type":"text/x-python","patch_set":2,"id":"43994c52_733ef45d","line":1282,"updated":"2024-10-21 14:28:06.000000000","message":"This can\u0027t be the right timestamp - we need to consider when the x-delete-at was *modified* not what value it was set to.","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"91057478d097c30f459c30200b9e7113c7896696","unresolved":false,"context_lines":[{"line_number":1279,"context_line":"                response \u003d HTTPPreconditionFailed("},{"line_number":1280,"context_line":"                    request\u003drequest,"},{"line_number":1281,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1282,"context_line":"                response.headers[\u0027X-Delete-At\u0027] \u003d orig_delete_at.normal"},{"line_number":1283,"context_line":"                return response"},{"line_number":1284,"context_line":"            else:"},{"line_number":1285,"context_line":"                # differentiate success from no object at all"}],"source_content_type":"text/x-python","patch_set":2,"id":"5d0f126e_732370cb","line":1282,"in_reply_to":"43994c52_733ef45d","updated":"2024-11-20 05:59:26.000000000","message":"Done","commit_id":"af2a9136fdb81fea4765b6267c60e272a0c35443"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6db377d65e1c47cb6fce345d26847e9edd5155b3","unresolved":true,"context_lines":[{"line_number":1225,"context_line":"            return False"},{"line_number":1226,"context_line":""},{"line_number":1227,"context_line":"        delete_task_ts_val \u003d request.headers[\u0027x-delete-task-timestamp\u0027]"},{"line_number":1228,"context_line":"        delete_task_ts \u003d Timestamp(delete_task_ts_val)"},{"line_number":1229,"context_line":"        metadata_ts \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027) or 0)"},{"line_number":1230,"context_line":"        if delete_task_ts \u003c metadata_ts:"},{"line_number":1231,"context_line":"            # this expirer queue entry is older than the time when the newer"}],"source_content_type":"text/x-python","patch_set":3,"id":"f52f8541_4423eb58","line":1228,"updated":"2025-01-15 18:45:37.000000000","message":"Shouldn\u0027t come up in practice, but might want a `try:`/`except ValueError: return False` here.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":false,"context_lines":[{"line_number":1225,"context_line":"            return False"},{"line_number":1226,"context_line":""},{"line_number":1227,"context_line":"        delete_task_ts_val \u003d request.headers[\u0027x-delete-task-timestamp\u0027]"},{"line_number":1228,"context_line":"        delete_task_ts \u003d Timestamp(delete_task_ts_val)"},{"line_number":1229,"context_line":"        metadata_ts \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027) or 0)"},{"line_number":1230,"context_line":"        if delete_task_ts \u003c metadata_ts:"},{"line_number":1231,"context_line":"            # this expirer queue entry is older than the time when the newer"}],"source_content_type":"text/x-python","patch_set":3,"id":"aed8a76e_ea967907","line":1228,"in_reply_to":"f52f8541_4423eb58","updated":"2025-01-22 17:02:02.000000000","message":"Honestly I think I prefer the http error than just silently ignoring the problem:\n\n```\ndiff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py\nindex bd390c80b..a1290e968 100644\n--- a/test/unit/obj/test_server.py\n+++ b/test/unit/obj/test_server.py\n@@ -7971,6 +7971,68 @@ class TestObjectController(BaseTestCase):\n         resp \u003d req.get_response(self.object_controller)\n         self.assertEqual(resp.status_int, 204)\n \n+    def test_invalid_delete_task_timestamp(self):\n+        create_object_ts \u003d next(self.ts)\n+        orig_delete_at \u003d utils.normalize_delete_at_timestamp(\n+            next(self.ts).normal)\n+        wrong_delete_at \u003d utils.normalize_delete_at_timestamp(\n+            next(self.ts).normal)\n+\n+        # Create an object with an an expiration time.\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},\n+            headers\u003dself._update_delete_at_headers({\n+                \u0027X-Timestamp\u0027: create_object_ts.normal,\n+                \u0027X-Delete-At\u0027: orig_delete_at,\n+                \u0027Content-Length\u0027: \u00274\u0027,\n+                \u0027Content-Type\u0027: \u0027application/octet-stream\u0027})\n+        )\n+        req.body \u003d \u0027TEST\u0027\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 201)\n+\n+        # if the expirer ever sent an invalid x-if-delete-at it\u0027d get a 400\n+        # until we fix the bug (good!)\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027,\n+            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},\n+            headers\u003d{\n+                \u0027X-Timestamp\u0027: Timestamp(orig_delete_at).normal,\n+                \u0027X-If-Delete-At\u0027: \u0027some-nonsense\u0027,\n+            }\n+        )\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 400)\n+        self.assertEqual(b\u0027Bad X-If-Delete-At header value\u0027, resp.body)\n+\n+        # if you send an *non-matching* x-if-delete-at AND some invalid\n+        # task-timestamps we blow up until we fix the bug (also fine)\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027,\n+            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},\n+            headers\u003d{\n+                \u0027X-Timestamp\u0027: Timestamp(orig_delete_at).normal,\n+                \u0027X-Backend-Expirer-Task-Timestamp\u0027: \u0027some-nonsense\u0027,\n+                \u0027X-If-Delete-At\u0027: wrong_delete_at,\n+            }\n+        )\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 500)\n+\n+        # if you send the *correct* x-delete-at the invalid task timestamp\n+        # doesn\u0027t even matter! (fair enough)\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027,\n+            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},\n+            headers\u003d{\n+                \u0027X-Timestamp\u0027: Timestamp(orig_delete_at).normal,\n+                \u0027X-Backend-Expirer-Task-Timestamp\u0027: \u0027some-nonsense\u0027,\n+                \u0027X-If-Delete-At\u0027: orig_delete_at,\n+            }\n+        )\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 204)\n+\n     def test_extra_headers_contain_object_bytes(self):\n         timestamp1 \u003d next(self.ts).normal\n         delete_at_timestamp1 \u003d int(time() + 1000)\n```\n\n\u003e Errors should never pass silently.\n\nI think this occasionally comes into contrast with the robustness principle; but I think with our own backend protocols it makes more sense to \"In the face of ambiguity, refuse the temptation to guess.\" and \"fail early fail loudly\"","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1227,"context_line":"        delete_task_ts_val \u003d request.headers[\u0027x-delete-task-timestamp\u0027]"},{"line_number":1228,"context_line":"        delete_task_ts \u003d Timestamp(delete_task_ts_val)"},{"line_number":1229,"context_line":"        metadata_ts \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027) or 0)"},{"line_number":1230,"context_line":"        if delete_task_ts \u003c metadata_ts:"},{"line_number":1231,"context_line":"            # this expirer queue entry is older than the time when the newer"},{"line_number":1232,"context_line":"            # x-delete-at got updated, it\u0027s stale and not needed."},{"line_number":1233,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":3,"id":"e19dde72_35acd64b","line":1230,"updated":"2024-12-14 00:32:10.000000000","message":"do we need to have *any* wiggle room here?\n\nIt probably doesn\u0027t matter since we only call this when the x-if-delete-at !\u003d x-if-delete-at - so we\u0027re going to return an error regardless.\n\nSince we\u0027re round-tripping through a container listing last_modified I worry we might clean-up a entry as \"stale\" when we should have \"just waited to try again later\"\n\nMaybe it helps that x-delete-at only allows 1s resolution.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":false,"context_lines":[{"line_number":1227,"context_line":"        delete_task_ts_val \u003d request.headers[\u0027x-delete-task-timestamp\u0027]"},{"line_number":1228,"context_line":"        delete_task_ts \u003d Timestamp(delete_task_ts_val)"},{"line_number":1229,"context_line":"        metadata_ts \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027) or 0)"},{"line_number":1230,"context_line":"        if delete_task_ts \u003c metadata_ts:"},{"line_number":1231,"context_line":"            # this expirer queue entry is older than the time when the newer"},{"line_number":1232,"context_line":"            # x-delete-at got updated, it\u0027s stale and not needed."},{"line_number":1233,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":3,"id":"d9e49b20_27a0a260","line":1230,"in_reply_to":"e19dde72_35acd64b","updated":"2025-01-22 17:02:02.000000000","message":"In retrospect think last_modified actually has just as much resolution as normal timestamps; and this isn\u0027t a \"float\" it\u0027s a *timestamp* so these kind of precise comparisons are ok by design.\n\nIf the object has had it\u0027s metadata updated *after* metadata update that created the x-delete-at task entry then the task entry is old/stale/invalid.  We could make that claim even if the x-delete-at values happened to match; but as the code is written this only matters after we\u0027ve already decided we\u0027re not going to write a tombstone.\n\nHere we\u0027re just trying to decide if in the opinion of this replica should the expirer retry this request and I don\u0027t see how this \"my metadata/x-delete-at is newer than the metadata of YOUR x-delete-at\" calculation could ever be wrong.  I think this is a fundamental observation of this optimization.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1254,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1255,"context_line":"        except DiskFileExpired as e:"},{"line_number":1256,"context_line":"            orig_timestamp \u003d e.timestamp"},{"line_number":1257,"context_line":"            orig_metadata \u003d e.metadata"},{"line_number":1258,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1259,"context_line":"        except DiskFileDeleted as e:"},{"line_number":1260,"context_line":"            orig_timestamp \u003d e.timestamp"}],"source_content_type":"text/x-python","patch_set":3,"id":"88337e69_48186314","line":1257,"updated":"2024-12-14 00:32:10.000000000","message":"so this is the case we\u0027re most interested in...\n\nIt looks like a DiskFileExpired\u0027s timestamp *is* the metadata.get(\u0027X-Timestamp\u0027)\n\nhttps://github.com/NVIDIA/swift/blob/master/swift/common/exceptions.py#L84-L93","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":1254,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1255,"context_line":"        except DiskFileExpired as e:"},{"line_number":1256,"context_line":"            orig_timestamp \u003d e.timestamp"},{"line_number":1257,"context_line":"            orig_metadata \u003d e.metadata"},{"line_number":1258,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1259,"context_line":"        except DiskFileDeleted as e:"},{"line_number":1260,"context_line":"            orig_timestamp \u003d e.timestamp"}],"source_content_type":"text/x-python","patch_set":3,"id":"39fdf45a_70cf6879","line":1257,"in_reply_to":"88337e69_48186314","updated":"2025-02-04 17:51:20.000000000","message":"cited Clay\u0027s own words, I think he is talking about this.\n```\nI\u0027m not sure I believe the stale-task-over-newer-tombstone-should-409\ncan be implemented reliably; and the current 412/404 response\n(depending on if the tombstones\u0027 been reclaimed?) is not incorrect and\ncould be optimized later if it\u0027s an issue.\n```","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1269,"context_line":"            if orig_timestamp \u003c req_timestamp:"},{"line_number":1270,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1271,"context_line":"            else:"},{"line_number":1272,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1273,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1274,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1275,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"15501a1b_94da29a2","line":1272,"updated":"2024-12-14 00:32:10.000000000","message":"oh actually client requests are subject to the same 409 restriction on DELETE older than PUT; but maybe you *can* delete from \"under\" a POST/meta_timestamp?","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":1269,"context_line":"            if orig_timestamp \u003c req_timestamp:"},{"line_number":1270,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1271,"context_line":"            else:"},{"line_number":1272,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1273,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1274,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1275,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"de034f3e_986d64b1","line":1272,"in_reply_to":"15501a1b_94da29a2","updated":"2025-01-22 17:02:02.000000000","message":"right, this doesn\u0027t look like the correct logic - it probably deserves at least a comment\n\nit reads like it\u0027s trying to say:\n\n```\nPUT.t0 \u003c DELETE.t1 -\u003e 204\nPUT.t1 \u003e\u003d DELETE.t0 -\u003e 409\n```\n\nBut I don\u0027t know why the same logic wouldn\u0027t apply to the `orig_metdata[\u0027X-Timestamp\u0027]`?\n\n```\nPUT.t0 \u003c POST.t1 \u003c DELETE.t2 -\u003e 204\nPOST.t2 \u003e PUT.t1 \u003e\u003d DELETE.t0 -\u003e 409\nPUT.t0 \u003c DELETE.t1 \u003c\u003d POST.t2 -\u003e 204 (!?)\n```\n\nMaybe the idea is there\u0027s some hidden PUT.t1+ that eventually shows up via replication that makes POST.t2 valid; I assume `cleanup_on_disk_files` will leave `.meta` files around over tombstones until the tombstone gets reclaimed or something.  OTOH, if there\u0027s NOT a PUT.t1+ file out there we better try and return 204 and write the tombstone?  I\u0027m not sure why that\u0027s better than error - we\u0027re already in this weird state where we\u0027re getting requests from a proxy who\u0027s writing \"old\" x-timestamps down.  Why do we ever need/want the DELETE to happen regardless of a newer x-timestamp on the .meta!?\n\nMaybe it\u0027s a requirement of SSYNC?\n\nnodeA: t0.data t3.meta\nnodeB: t1.ts\n\nThe correct resolution of *that* state must be t1.ts (?)\n\nBut if we\u0027re just \"missing\" the latest .data?\n\nnodeA: t0.data t3.meta\nnodeB: t1.ts\nnodeC: t2.data\n\nThe final resolution of that state *should* be (t2.data, t3.meta) - so I assume if nodeB syncs to nodeA it should result in (t1.ts, t3.meta) - but I don\u0027t think that\u0027s what our diskfile tests say would happen:\n\nhttps://github.com/NVIDIA/swift/blob/master/test/unit/obj/test_diskfile.py#L2272-L2279","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1dd654bcd4ef95aecd182f884913f28cd899c993","unresolved":true,"context_lines":[{"line_number":1269,"context_line":"            if orig_timestamp \u003c req_timestamp:"},{"line_number":1270,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1271,"context_line":"            else:"},{"line_number":1272,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1273,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1274,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1275,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff095dea_0c603295","line":1272,"in_reply_to":"de034f3e_986d64b1","updated":"2025-01-22 18:36:16.000000000","message":"IMHO it is correct that a DELETE at t1 succeeds (204) regardless of a meta file timestamp at t2.\n\nActions should always play out in the order that requests (would) have been received. The fact that a POST at t2 somehow got ahead of a DELETE at t1 does not make the POST negate the DELETE.\n\nThe diskfile tests are hard to grok but IIRC cleanup_ondisk_files will leave an isolated meta file on disk until reclaim_age in case an older data file shows up.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8a651faac1f88a0a498ed6b65e774fdaf3bb3b1f","unresolved":false,"context_lines":[{"line_number":1269,"context_line":"            if orig_timestamp \u003c req_timestamp:"},{"line_number":1270,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1271,"context_line":"            else:"},{"line_number":1272,"context_line":"                response_class \u003d HTTPConflict"},{"line_number":1273,"context_line":"        response_timestamp \u003d max(orig_timestamp, req_timestamp)"},{"line_number":1274,"context_line":"        orig_delete_at \u003d Timestamp(orig_metadata.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":1275,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"21c59e71_aa81ed67","line":1272,"in_reply_to":"ff095dea_0c603295","updated":"2025-01-27 23:54:34.000000000","message":"yes, this is the case. If there are any .meta file but no .data file, ``cleanup_ondisk_files`` will store this meta file in the “possible reclaim” list: \nhttps://github.com/NVIDIA/swift/blob/master/swift/obj/diskfile.py#L3118\n\nand later ``cleanup_ondisk_files`` checks this meta file\u0027s timestamp against the reclaim_age; if this meta file is old enough, ``cleanup_ondisk_files`` adds it to the obsolete list and then deletes it. Otherwise, the file remains on disk:\nhttps://github.com/NVIDIA/swift/blob/master/swift/obj/diskfile.py#L1109","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1281,"context_line":"            return HTTPBadRequest("},{"line_number":1282,"context_line":"                request\u003drequest,"},{"line_number":1283,"context_line":"                body\u003d\u0027Bad X-If-Delete-At header value\u0027)"},{"line_number":1284,"context_line":"        else:"},{"line_number":1285,"context_line":"            # request includes x-if-delete-at; we must not place a tombstone"},{"line_number":1286,"context_line":"            # if we can not verify the x-if-delete-at time"},{"line_number":1287,"context_line":"            if not orig_timestamp:"}],"source_content_type":"text/x-python","patch_set":3,"id":"b1d4d085_b4913724","line":1284,"updated":"2024-12-14 00:32:10.000000000","message":"ok, so all of this block is only for handling `req_if_delete_at`; none of these conditions/behaviors effect client requests.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":false,"context_lines":[{"line_number":1281,"context_line":"            return HTTPBadRequest("},{"line_number":1282,"context_line":"                request\u003drequest,"},{"line_number":1283,"context_line":"                body\u003d\u0027Bad X-If-Delete-At header value\u0027)"},{"line_number":1284,"context_line":"        else:"},{"line_number":1285,"context_line":"            # request includes x-if-delete-at; we must not place a tombstone"},{"line_number":1286,"context_line":"            # if we can not verify the x-if-delete-at time"},{"line_number":1287,"context_line":"            if not orig_timestamp:"}],"source_content_type":"text/x-python","patch_set":3,"id":"0bf45a03_4533e33f","line":1284,"in_reply_to":"b1d4d085_b4913724","updated":"2025-01-22 17:02:02.000000000","message":"well, except Tim confirmed that clients *can* send x-if-delete-at if they wanted to for some reason...","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1287,"context_line":"            if not orig_timestamp:"},{"line_number":1288,"context_line":"                # no object found at all"},{"line_number":1289,"context_line":"                return HTTPNotFound()"},{"line_number":1290,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1291,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1292,"context_line":"                return HTTPConflict()"},{"line_number":1293,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"}],"source_content_type":"text/x-python","patch_set":3,"id":"bb53aa6b_7ae8abde","line":1290,"updated":"2024-12-14 00:32:10.000000000","message":"I\u0027m actually surprised this condition doesn\u0027t apply to user requests from slow proxies....\n\nturns out: it does; we just spell it out twice.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":false,"context_lines":[{"line_number":1287,"context_line":"            if not orig_timestamp:"},{"line_number":1288,"context_line":"                # no object found at all"},{"line_number":1289,"context_line":"                return HTTPNotFound()"},{"line_number":1290,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1291,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1292,"context_line":"                return HTTPConflict()"},{"line_number":1293,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"}],"source_content_type":"text/x-python","patch_set":3,"id":"490ca2c5_8e7f31a3","line":1290,"in_reply_to":"bb53aa6b_7ae8abde","updated":"2025-01-22 17:02:02.000000000","message":"I think we could \"just\" assert the response_class here is already HTTPConflict always.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":1296,"context_line":"                return HTTPPreconditionFailed("},{"line_number":1297,"context_line":"                    request\u003drequest,"},{"line_number":1298,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1299,"context_line":"            else:"},{"line_number":1300,"context_line":"                # differentiate success from no object at all"},{"line_number":1301,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1302,"context_line":"        self._conditional_delete_at_update("}],"source_content_type":"text/x-python","patch_set":3,"id":"c0577aad_1d5beb53","line":1299,"updated":"2024-12-14 00:32:10.000000000","message":"i\u0027m struggling to think through a stale request that just happens to have the same orig_delete_at timestamp; it\u0027s certainly possible to do multiple POSTs with the same `x-delete-at: t2`; it\u0027s *probably* reasonable to handle the first one that shows up that matches (even if it was created from a POST at t0 and our metadata says our x-delete-at was created from a POST at t1)","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":false,"context_lines":[{"line_number":1296,"context_line":"                return HTTPPreconditionFailed("},{"line_number":1297,"context_line":"                    request\u003drequest,"},{"line_number":1298,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1299,"context_line":"            else:"},{"line_number":1300,"context_line":"                # differentiate success from no object at all"},{"line_number":1301,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1302,"context_line":"        self._conditional_delete_at_update("}],"source_content_type":"text/x-python","patch_set":3,"id":"b670a37d_ea652d76","line":1299,"in_reply_to":"8fd03c99_24a52c52","updated":"2025-02-04 21:51:11.000000000","message":"Squashed the test case, thanks!","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":1296,"context_line":"                return HTTPPreconditionFailed("},{"line_number":1297,"context_line":"                    request\u003drequest,"},{"line_number":1298,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1299,"context_line":"            else:"},{"line_number":1300,"context_line":"                # differentiate success from no object at all"},{"line_number":1301,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1302,"context_line":"        self._conditional_delete_at_update("}],"source_content_type":"text/x-python","patch_set":3,"id":"8fd03c99_24a52c52","line":1299,"in_reply_to":"c0577aad_1d5beb53","updated":"2025-01-22 17:02:02.000000000","message":"the reason it *might* be better to process the task with the *matching* meta-timestamp/task-timestamp might depend on how the expirer deals with 404s who\u0027s x-timestamp\u003ex-task-timestamp.\n\n```\n+    def test_stale_delete_task_timestamp_404(self):\n+        task_timestamp_ts \u003d create_object_ts \u003d next(self.ts)\n+        user_deleted_at \u003d next(self.ts)\n+        orig_delete_at \u003d utils.normalize_delete_at_timestamp(\n+            next(self.ts).normal)\n+\n+        # First, create an object with an an expiration time.\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},\n+            headers\u003dself._update_delete_at_headers({\n+                \u0027X-Timestamp\u0027: create_object_ts.normal,\n+                \u0027X-Delete-At\u0027: orig_delete_at,\n+                \u0027Content-Length\u0027: \u00274\u0027,\n+                \u0027Content-Type\u0027: \u0027application/octet-stream\u0027})\n+        )\n+        req.body \u003d \u0027TEST\u0027\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 201)\n+\n+        # before the expirer gets a chance to do anything DELETE it!\n+        req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027, method\u003d\u0027DELETE\u0027,\n+                            headers\u003d{\n+                                \u0027X-Timestamp\u0027: user_deleted_at.normal,\n+                            })\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 204)\n+        \n+        # ... but the object *is* a tombstone\n+        req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027, method\u003d\u0027HEAD\u0027)\n+        resp \u003d req.get_response(self.object_controller)\n+        self.assertEqual(resp.status_int, 404)\n+        self.assertEqual(user_deleted_at,\n+                         Timestamp(resp.headers[\u0027X-Backend-Timestamp\u0027]))\n+\n+        # when the exprier processes the task-timestamp; it should be \"stale\"\n+        req \u003d Request.blank(\n+            \u0027/sda1/p/a/c/o\u0027,\n+            method\u003d\u0027DELETE\u0027,\n+            headers\u003d{\n+                \u0027X-Timestamp\u0027: Timestamp(orig_delete_at).normal,\n+                \u0027X-Backend-Expirer-Task-Timestamp\u0027: task_timestamp_ts.normal,\n+                \u0027X-If-Delete-At\u0027: orig_delete_at,\n+            }\n+        )\n+        resp \u003d req.get_response(self.object_controller)\n+        # this should 409 right?\n+        self.assertEqual(resp.status_int, 412)\n+        # this is unfortunate\n+        self.assertNotIn(\u0027X-Backend-Timestamp\u0027, resp.headers)\n+\n+        # ... and the expirer will \"only\" keep trying until the async_pending\n+        # for the DELETE into the .expiring objects account gets processed to\n+        # wipe out the stale task.  But if the obj-server returned 409 (because\n+        # it\u0027s metdata knows it\u0027s newer than task-timestamp) - the expirer\n+        # could reap the task immediately.\n+\n```\n\nSo this test doesn\u0027t directly demonstrate two POST requests with the same x-delete-at; nor the expirer handling one vs the other - but it does show that after an object is deleted (in this case by the user) the expirer will get a 412 (!) because the `orig_delete_at !\u003d req_if_delete_at` in the case of a `except DiskFileNotExist` because `orig_delete_at \u003d Timestamp({}.get(\u0027X-Delete-At\u0027) or 0)`\n\n... so the expirer will keep trying until a reclaim age (or the task cleanup async pending gets processed).  But if the task cleanup goes to the wrong container... then we\u0027re getting 412s for a reclaim age!\n\nIF we made the decision about `current_meta_timestamp \u003e task_timestamp` and returned a 409 *first* (before even checking if x-delete-at \u003d\u003d x-if-delete-at) we could reap stale entries and not \"accidently\" cause an object to 404 \"just\" because our task happened to match some OTHER tasks x-if-delete-at).  I don\u0027t know that would be *strictly* better - but it would not result in 412s for a relcaim.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":1298,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1299,"context_line":"            else:"},{"line_number":1300,"context_line":"                # differentiate success from no object at all"},{"line_number":1301,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1302,"context_line":"        self._conditional_delete_at_update("},{"line_number":1303,"context_line":"            request, device, account, container, obj, policy, {},"},{"line_number":1304,"context_line":"            orig_delete_at, 0"}],"source_content_type":"text/x-python","patch_set":3,"id":"b1c61a26_0697061d","line":1301,"updated":"2025-01-22 17:02:02.000000000","message":"of course if we hadn\u0027t done the early return on conflict above then we\u0027d reset response_class here so all of this would have to change...\n\nthis (existing) code reads sort of messy IMHO.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":false,"context_lines":[{"line_number":1298,"context_line":"                    body\u003d\u0027X-If-Delete-At and X-Delete-At do not match\u0027)"},{"line_number":1299,"context_line":"            else:"},{"line_number":1300,"context_line":"                # differentiate success from no object at all"},{"line_number":1301,"context_line":"                response_class \u003d HTTPNoContent"},{"line_number":1302,"context_line":"        self._conditional_delete_at_update("},{"line_number":1303,"context_line":"            request, device, account, container, obj, policy, {},"},{"line_number":1304,"context_line":"            orig_delete_at, 0"}],"source_content_type":"text/x-python","patch_set":3,"id":"a92b3702_9232168f","line":1301,"in_reply_to":"b1c61a26_0697061d","updated":"2025-02-04 21:51:11.000000000","message":"Acknowledged","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":1269,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1270,"context_line":"        except DiskFileDeleted as e:"},{"line_number":1271,"context_line":"            orig_timestamp \u003d e.timestamp"},{"line_number":1272,"context_line":"            orig_metadata \u003d {}"},{"line_number":1273,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1274,"context_line":"        except (DiskFileNotExist, DiskFileQuarantined):"},{"line_number":1275,"context_line":"            orig_timestamp \u003d 0"}],"source_content_type":"text/x-python","patch_set":9,"id":"48cc3cfb_5947518c","line":1272,"updated":"2025-01-22 17:02:02.000000000","message":"it might be better if our `is_delete_task_stale` helper just accepted the `orig_timestamp` instead of the `orig_metdata`\n\nIf it did it could make the same logical decision about when \"stale\" task-timestamps are overwriten by DELETE as it\u0027s now added for PUT/POST overwrites.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":true,"context_lines":[{"line_number":1269,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1270,"context_line":"        except DiskFileDeleted as e:"},{"line_number":1271,"context_line":"            orig_timestamp \u003d e.timestamp"},{"line_number":1272,"context_line":"            orig_metadata \u003d {}"},{"line_number":1273,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1274,"context_line":"        except (DiskFileNotExist, DiskFileQuarantined):"},{"line_number":1275,"context_line":"            orig_timestamp \u003d 0"}],"source_content_type":"text/x-python","patch_set":9,"id":"8d332396_6f9062ec","line":1272,"in_reply_to":"48cc3cfb_5947518c","updated":"2025-02-04 21:51:11.000000000","message":"``orig_timestamp`` here will be the `x-timestamp` of the tombstone file. it won\u0027t be used for the \u0027x-delete-at\u0027 handling since \u0027x-delete-at\u0027 will be regarded as 0.\nhttps://github.com/NVIDIA/swift/blob/master/swift/obj/diskfile.py#L2682-L2708\nhttps://github.com/NVIDIA/swift/blob/master/swift/common/exceptions.py#L84-L93\n\n\nBut for the normal case, we shouldn\u0027t use it: with PUT.t0 and POST.t1 and before object is deleted,``orig_timestamp \u003d disk_file.data_timestamp``, so ``orig_timestamp`` is going to be t0, while ``orig_metdata.get(\u0027x-timestamp\u0027)`` will be t1, and t1 is what we need.\n\n\nwhen I switch to use orig_timestamp, our intended 409 responses will become 412:\nhttps://review.opendev.org/c/openstack/swift/+/932447/8/test/unit/obj/test_server.py#7927","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":1272,"context_line":"            orig_metadata \u003d {}"},{"line_number":1273,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1274,"context_line":"        except (DiskFileNotExist, DiskFileQuarantined):"},{"line_number":1275,"context_line":"            orig_timestamp \u003d 0"},{"line_number":1276,"context_line":"            orig_metadata \u003d {}"},{"line_number":1277,"context_line":"            response_class \u003d HTTPNotFound"},{"line_number":1278,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f378eba_2b5e6e00","line":1275,"updated":"2025-01-22 17:02:02.000000000","message":"this is why we\u0027d need a *real* 404/not-found to NOT accidently return 409\n\nif we have NO tombstone we can\u0027t say for sure if the task-timestamp is older than our more recent update/tombstone.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1dd654bcd4ef95aecd182f884913f28cd899c993","unresolved":true,"context_lines":[{"line_number":1298,"context_line":"            if not orig_timestamp:"},{"line_number":1299,"context_line":"                # no object found at all"},{"line_number":1300,"context_line":"                return HTTPNotFound()"},{"line_number":1301,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1302,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1303,"context_line":"                return HTTPConflict()"},{"line_number":1304,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"}],"source_content_type":"text/x-python","patch_set":9,"id":"0315914c_fd933ada","line":1301,"range":{"start_line":1301,"start_character":30,"end_line":1301,"end_character":32},"updated":"2025-01-22 18:36:16.000000000","message":"would be clearer if this was at least written with a ``\u003c`` as per line 1280, but perhaps could also be written \n\n```\nif response_class \u003d\u003d HTTPConflict:\n    return response_class()\n```","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":1298,"context_line":"            if not orig_timestamp:"},{"line_number":1299,"context_line":"                # no object found at all"},{"line_number":1300,"context_line":"                return HTTPNotFound()"},{"line_number":1301,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1302,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1303,"context_line":"                return HTTPConflict()"},{"line_number":1304,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"}],"source_content_type":"text/x-python","patch_set":9,"id":"7e2064c6_aad16bdb","line":1301,"range":{"start_line":1301,"start_character":30,"end_line":1301,"end_character":32},"in_reply_to":"0315914c_fd933ada","updated":"2025-02-04 17:51:20.000000000","message":"ACK, this could be a follow-up patch to refactor the existing code.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1dd654bcd4ef95aecd182f884913f28cd899c993","unresolved":true,"context_lines":[{"line_number":1300,"context_line":"                return HTTPNotFound()"},{"line_number":1301,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1302,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1303,"context_line":"                return HTTPConflict()"},{"line_number":1304,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"},{"line_number":1305,"context_line":"                if self._is_delete_task_stale(request, orig_metadata):"},{"line_number":1306,"context_line":"                    return HTTPConflict()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5133eeed_36f6f0a1","line":1303,"updated":"2025-01-22 18:36:16.000000000","message":"looks like these early returns are here *before* an \u0027x-if-delete-at\u0027 request might modify the response type.\n\nIt might be clearer if they returned sooner, but there\u0027s some difference in the population of headers w.r.t. a non \u0027x-if-delete-at\u0027 request","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":1300,"context_line":"                return HTTPNotFound()"},{"line_number":1301,"context_line":"            if orig_timestamp \u003e\u003d req_timestamp:"},{"line_number":1302,"context_line":"                # Found a newer object -- return 409 as work item is stale"},{"line_number":1303,"context_line":"                return HTTPConflict()"},{"line_number":1304,"context_line":"            if orig_delete_at !\u003d req_if_delete_at:"},{"line_number":1305,"context_line":"                if self._is_delete_task_stale(request, orig_metadata):"},{"line_number":1306,"context_line":"                    return HTTPConflict()"}],"source_content_type":"text/x-python","patch_set":9,"id":"ebf40e49_3320bc0f","line":1303,"in_reply_to":"5133eeed_36f6f0a1","updated":"2025-02-04 17:51:20.000000000","message":"ACK, this also could be a follow-up patch to refactor the existing code.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"}],"test/probe/test_object_expirer.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":927,"context_line":"        # shutdown *some object* servers (keep majority online so POST works)"},{"line_number":928,"context_line":"        brain.stop_handoff_half()"},{"line_number":929,"context_line":"        # update the object before it expires!"},{"line_number":930,"context_line":"        new_delete_at \u003d max(orig_delete_at, int(time.time())) + 1"},{"line_number":931,"context_line":"        client.post_object("},{"line_number":932,"context_line":"            self.url, self.token, self.container_name, self.object_name,"},{"line_number":933,"context_line":"            headers\u003d{\u0027X-Delete-At\u0027: new_delete_at})"}],"source_content_type":"text/x-python","patch_set":9,"id":"134f2777_6099613b","line":930,"updated":"2025-01-22 17:02:02.000000000","message":"i remember struggling with this value in testing\n\nyou need the object to NOT be expired by the time you go to update it\u0027s x-delete-at or you get a 404 on your POST\n\nMAYBE it doesn\u0027t matter if the \"to be cleaned up value\" is WAY out in the future; but for some reason I wanted it to be LESS than the new/updated delete-at so that the expirer will process the stale one before it successfully processes the new value\n\nActually, if we did a different test for the other way around - the stale entry just gets a 404, which can still end up sticking around for a reclaim age.  I guess we could add a similar fix in the object-server?","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd9ab4918b197f8f1412a969805b06cfe70d8a63","unresolved":true,"context_lines":[{"line_number":959,"context_line":"                          \u0027 but still readable after %s\u0027 % ("},{"line_number":960,"context_line":"                              self.object_name, self.container_name,"},{"line_number":961,"context_line":"                              new_delete_at, timeout))"},{"line_number":962,"context_line":"            time.sleep(1.0)"},{"line_number":963,"context_line":""},{"line_number":964,"context_line":"        # it\u0027s basically terrifying that if we don\u0027t run the object"},{"line_number":965,"context_line":"        # replicators here first, the stale entry might \"work\" on the"}],"source_content_type":"text/x-python","patch_set":9,"id":"9c28b558_c27411fa","line":962,"updated":"2025-01-22 17:02:02.000000000","message":"this sleep might be needlessly long\n\n```\nvagrant@saio:~$ time pytest swift/test/probe/test_object_expirer.py::TestObjectExpirer::test_expirer_outdated_task_timestamp_409\n...\nreal    0m49.328s\n```\n\nwith `sleep(0.1)`\n\n```\nvagrant@saio:~$ time pytest swift/test/probe/test_object_expirer.py::TestObjectExpirer::test_expirer_outdated_task_timestamp_409\n...\nreal    0m44.153s\n```\n\nI guess the runtime is mostly dominated by other factors.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":959,"context_line":"                          \u0027 but still readable after %s\u0027 % ("},{"line_number":960,"context_line":"                              self.object_name, self.container_name,"},{"line_number":961,"context_line":"                              new_delete_at, timeout))"},{"line_number":962,"context_line":"            time.sleep(1.0)"},{"line_number":963,"context_line":""},{"line_number":964,"context_line":"        # it\u0027s basically terrifying that if we don\u0027t run the object"},{"line_number":965,"context_line":"        # replicators here first, the stale entry might \"work\" on the"}],"source_content_type":"text/x-python","patch_set":9,"id":"f1699cd1_54b3c28b","line":962,"in_reply_to":"9c28b558_c27411fa","updated":"2025-02-11 03:52:45.000000000","message":"yeah its the resetswift takes a bunch of time.","commit_id":"782e610b5c045f4f880b15c7abdead11e39543e2"}],"test/unit/obj/test_expirer.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":3141,"context_line":"            self.expirer.run_once()"},{"line_number":3142,"context_line":""},{"line_number":3143,"context_line":"        # executed tasks are with past time"},{"line_number":3144,"context_line":"        past_time_task_creation \u003d Timestamp(float(self.past_time) - 86400)"},{"line_number":3145,"context_line":"        just_past_time_task_creation \u003d Timestamp("},{"line_number":3146,"context_line":"            float(self.just_past_time) - 86400)"},{"line_number":3147,"context_line":"        self.assertEqual("}],"source_content_type":"text/x-python","patch_set":12,"id":"4e9686b1_c8312998","line":3144,"range":{"start_line":3144,"start_character":68,"end_line":3144,"end_character":73},"updated":"2025-02-11 03:52:45.000000000","message":"This number comes up alot. It\u0027s obvioulsy a day (24 \\* 60 \\* 60).","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":3500,"context_line":"    def test_delete_actual_object_quotes(self):"},{"line_number":3501,"context_line":"        name \u003d \u0027this name/should get/quoted\u0027"},{"line_number":3502,"context_line":"        timestamp \u003d Timestamp(\u00271366063156.863045\u0027)"},{"line_number":3503,"context_line":"        task_ts \u003d Timestamp(\u00271365976756.863045\u0027)"},{"line_number":3504,"context_line":"        x \u003d expirer.ObjectExpirer({}, swift\u003dself.make_fake_ic(self.fake_swift))"},{"line_number":3505,"context_line":"        x.swift.make_request \u003d mock.Mock()"},{"line_number":3506,"context_line":"        x.swift.make_request.return_value.status_int \u003d 204"}],"source_content_type":"text/x-python","patch_set":12,"id":"0d1ddcb8_665695fe","line":3503,"updated":"2025-02-11 03:52:45.000000000","message":"Seeing as we keep using 86400 for a day all the time, can we just be consistant and:\n\n    ts \u003d 1366063156.863045\n    timestamp \u003d Timestamp(ts)\n    task_ts \u003d Timestamp(ts - 86400)\n    \nOr a comment or something about being a day before.","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":3538,"context_line":"            }"},{"line_number":3539,"context_line":"        )"},{"line_number":3540,"context_line":""},{"line_number":3541,"context_line":"        # Create a FakeSwift client and nitialize ObjectExpirer with it"},{"line_number":3542,"context_line":"        ic \u003d internal_client.InternalClient(None, \u0027test-ic\u0027, 1, app\u003dfake_swift)"},{"line_number":3543,"context_line":"        x \u003d expirer.ObjectExpirer({}, swift\u003dic)"},{"line_number":3544,"context_line":"        delete_at_ts \u003d Timestamp(\u00271234\u0027)"}],"source_content_type":"text/x-python","patch_set":12,"id":"216919f2_a5f7a056","line":3541,"range":{"start_line":3541,"start_character":40,"end_line":3541,"end_character":49},"updated":"2025-02-11 03:52:45.000000000","message":"initialize","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":7742,"context_line":"        self.assertEqual(resp.status_int, 400)"},{"line_number":7743,"context_line":""},{"line_number":7744,"context_line":"    def test_DELETE_delete_task_timestamp(self):"},{"line_number":7745,"context_line":"        test_time \u003d time() + 10000"},{"line_number":7746,"context_line":"        # Create an object with an an expiration time."},{"line_number":7747,"context_line":"        delete_at_timestamp \u003d int(test_time - 1)"},{"line_number":7748,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":3,"id":"75ed77e4_bba34f40","line":7745,"updated":"2024-12-14 00:32:10.000000000","message":"I\u0027m having a bit of trouble reasoning about this whole test acting as if it\u0027s running \"in the future\" and all of the timestamps being created as some variation of \"slightly less in the future\" ... like I need a numberline to figure out the timeline.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":7742,"context_line":"        self.assertEqual(resp.status_int, 400)"},{"line_number":7743,"context_line":""},{"line_number":7744,"context_line":"    def test_DELETE_delete_task_timestamp(self):"},{"line_number":7745,"context_line":"        test_time \u003d time() + 10000"},{"line_number":7746,"context_line":"        # Create an object with an an expiration time."},{"line_number":7747,"context_line":"        delete_at_timestamp \u003d int(test_time - 1)"},{"line_number":7748,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":3,"id":"8f68866f_0da1f42a","line":7745,"in_reply_to":"75ed77e4_bba34f40","updated":"2025-02-04 17:51:20.000000000","message":"Acknowledged","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":7782,"context_line":"        req \u003d Request.blank("},{"line_number":7783,"context_line":"            \u0027/sda1/p/a/c/o\u0027,"},{"line_number":7784,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7785,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7786,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7787,"context_line":"                     normalize_timestamp(test_time - 50),"},{"line_number":7788,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"}],"source_content_type":"text/x-python","patch_set":3,"id":"25c0219f_24790693","line":7785,"updated":"2024-12-14 00:32:10.000000000","message":"I believe the expirer will use the x-delete-at timestamp as x-timestamp of it\u0027s request:\n\nhttps://github.com/NVIDIA/swift/blob/master/swift/obj/expirer.py#L612","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":7782,"context_line":"        req \u003d Request.blank("},{"line_number":7783,"context_line":"            \u0027/sda1/p/a/c/o\u0027,"},{"line_number":7784,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7785,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7786,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7787,"context_line":"                     normalize_timestamp(test_time - 50),"},{"line_number":7788,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"}],"source_content_type":"text/x-python","patch_set":3,"id":"415b2106_2dea9baf","line":7785,"in_reply_to":"25c0219f_24790693","updated":"2025-02-04 17:51:20.000000000","message":"Acknowledged","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":7784,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7785,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7786,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7787,"context_line":"                     normalize_timestamp(test_time - 50),"},{"line_number":7788,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"},{"line_number":7789,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":7790,"context_line":"        self.assertEqual(resp.status_int, 409)"}],"source_content_type":"text/x-python","patch_set":3,"id":"728da393_81dd23ac","line":7787,"updated":"2024-12-14 00:32:10.000000000","message":"I can see this is the x-timestamp of the original PUT request that set this x-delete-at; but it might be better if we used a named variable in both requests.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":7784,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7785,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7786,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7787,"context_line":"                     normalize_timestamp(test_time - 50),"},{"line_number":7788,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"},{"line_number":7789,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":7790,"context_line":"        self.assertEqual(resp.status_int, 409)"}],"source_content_type":"text/x-python","patch_set":3,"id":"1867038f_29617731","line":7787,"in_reply_to":"728da393_81dd23ac","updated":"2025-02-04 17:51:20.000000000","message":"Done","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a677b62de63f8fb8a0d563d0bdac64ad1f3dd04d","unresolved":true,"context_lines":[{"line_number":7796,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7797,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7798,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7799,"context_line":"                     normalize_timestamp(test_time - 49),"},{"line_number":7800,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"},{"line_number":7801,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":7802,"context_line":"        self.assertEqual(resp.status_int, 412)"}],"source_content_type":"text/x-python","patch_set":3,"id":"c2a5e436_ce1b4d14","line":7799,"updated":"2024-12-14 00:32:10.000000000","message":"i don\u0027t understand, this is the same timestamp as the POST that changed the x-delete-at to `new_delete_at_timestamp`; so all that matters here is the `x-if-delete-at` doesn\u0027t match.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":false,"context_lines":[{"line_number":7796,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":7797,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(test_time - 20),"},{"line_number":7798,"context_line":"                     \u0027X-Delete-Task-Timestamp\u0027:"},{"line_number":7799,"context_line":"                     normalize_timestamp(test_time - 49),"},{"line_number":7800,"context_line":"                     \u0027X-If-Delete-At\u0027: delete_at_timestamp})"},{"line_number":7801,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":7802,"context_line":"        self.assertEqual(resp.status_int, 412)"}],"source_content_type":"text/x-python","patch_set":3,"id":"53786ed2_baf16487","line":7799,"in_reply_to":"c2a5e436_ce1b4d14","updated":"2025-02-04 17:51:20.000000000","message":"not related anymore.","commit_id":"6954afe1adb58957fa7a453dc3771b65c071d69d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ae4e1b83070edc5e60fcdd6ce4ae07275cc168e5","unresolved":false,"context_lines":[{"line_number":8033,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":8034,"context_line":"        self.assertEqual(resp.status_int, 204)"},{"line_number":8035,"context_line":""},{"line_number":8036,"context_line":"    def test_stale_delete_task_timestamp_404(self):"},{"line_number":8037,"context_line":"        task_timestamp_ts \u003d create_object_ts \u003d next(self.ts)"},{"line_number":8038,"context_line":"        user_deleted_at \u003d next(self.ts)"},{"line_number":8039,"context_line":"        orig_delete_at \u003d utils.normalize_delete_at_timestamp("}],"source_content_type":"text/x-python","patch_set":10,"id":"05f06b47_c6d385a4","line":8036,"updated":"2025-02-04 21:51:11.000000000","message":"this test case passed when I reverted the changes in this patch.","commit_id":"0399ccea2d9b2701b75b0f79b6b7f51e72fe7a56"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e039f2977c442d7e5d49039522a7af0b9dfa8dc9","unresolved":true,"context_lines":[{"line_number":8122,"context_line":"        # tombstone file left, and won\u0027t return \u0027X-Delete-At\u0027 header anymore"},{"line_number":8123,"context_line":"        # when openning the data file;"},{"line_number":8124,"context_line":"        #"},{"line_number":8125,"context_line":"        # NOTE: ``disk_file.read_metadata`` returns ``DiskFileDeleted`` during"},{"line_number":8126,"context_line":"        # object server DELETE handling, but ``HTTPNotFound`` will be ignored"},{"line_number":8127,"context_line":"        # during the following ``x-delete-at`` handling, this probably should"},{"line_number":8128,"context_line":"        # be fixed and ideally object server DELETE should return 404."}],"source_content_type":"text/x-python","patch_set":10,"id":"b03c9bbf_a75fd150","line":8125,"updated":"2025-02-04 17:51:20.000000000","message":"probably we should create an upstream bug on this?","commit_id":"0399ccea2d9b2701b75b0f79b6b7f51e72fe7a56"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":7802,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":7803,"context_line":"        self.assertEqual(resp.status_int, 204)"},{"line_number":7804,"context_line":""},{"line_number":7805,"context_line":"        # Create an object with an an expiration time."},{"line_number":7806,"context_line":"        delete_at_timestamp \u003d int(test_time - 1)"},{"line_number":7807,"context_line":"        req \u003d Request.blank("},{"line_number":7808,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"}],"source_content_type":"text/x-python","patch_set":12,"id":"d0268beb_dd143154","line":7805,"range":{"start_line":7805,"start_character":35,"end_line":7805,"end_character":37},"updated":"2025-02-11 03:52:45.000000000","message":"s/an//","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c59d66e1260a4fa2bafae3882027a7ae6b80c2e4","unresolved":true,"context_lines":[{"line_number":7855,"context_line":"        new_delete_at \u003d utils.normalize_delete_at_timestamp("},{"line_number":7856,"context_line":"            next(self.ts).normal)"},{"line_number":7857,"context_line":""},{"line_number":7858,"context_line":"        # Create an object with an an expiration time."},{"line_number":7859,"context_line":"        req \u003d Request.blank("},{"line_number":7860,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":7861,"context_line":"            headers\u003dself._update_delete_at_headers({"}],"source_content_type":"text/x-python","patch_set":12,"id":"9ac55358_7a8c1567","line":7858,"range":{"start_line":7858,"start_character":35,"end_line":7858,"end_character":37},"updated":"2025-02-11 03:52:45.000000000","message":"s/an//","commit_id":"5b70606cc8d5734af7f88a297921514498a0f982"}]}