)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"b6fb8668d7a23afaa4284824efd3e2c261f3b900","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"df05c67a_29bf0d6e","updated":"2025-01-21 16:44:44.000000000","message":"Thanks for helping out, I will squash the changes.","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"}],"swift/obj/expirer.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"b6fb8668d7a23afaa4284824efd3e2c261f3b900","unresolved":true,"context_lines":[{"line_number":665,"context_line":"                is_async \u003d o[\u0027content_type\u0027] \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":666,"context_line":"                last_modified_str \u003d o[\u0027last_modified\u0027]"},{"line_number":667,"context_line":"            except KeyError:"},{"line_number":668,"context_line":"                import pdb"},{"line_number":669,"context_line":"                pdb.set_trace()"},{"line_number":670,"context_line":"                self.logger.exception(\u0027Unexcepted error handling task %r\u0027 %"},{"line_number":671,"context_line":"                                      task_object)"}],"source_content_type":"text/x-python","patch_set":2,"id":"c87d7aa8_ba8a7523","line":668,"updated":"2025-01-21 16:44:44.000000000","message":"I will remove this debug breakpoint","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"b6fb8668d7a23afaa4284824efd3e2c261f3b900","unresolved":true,"context_lines":[{"line_number":926,"context_line":"                self.logger.increment(\u0027segments\u0027)"},{"line_number":927,"context_line":""},{"line_number":928,"context_line":"    def delete_actual_object(self, actual_obj, timestamp,"},{"line_number":929,"context_line":"                             is_async_delete, task_timestamp):"},{"line_number":930,"context_line":"        \"\"\""},{"line_number":931,"context_line":"        Deletes the end-user object indicated by the actual object name given"},{"line_number":932,"context_line":"        \u0027\u003caccount\u003e/\u003ccontainer\u003e/\u003cobject\u003e\u0027."}],"source_content_type":"text/x-python","patch_set":2,"id":"ee079512_5ff2ec05","line":929,"updated":"2025-01-21 16:44:44.000000000","message":"I agree, this is much better, given we know ``task_timestamp`` won\u0027t be None at here.","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"}],"test/probe/test_object_expirer.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ecf96f00a08fdf95c77b96314c7be61eef0727f4","unresolved":true,"context_lines":[{"line_number":937,"context_line":"                async_updates.append(pickle.load(f))"},{"line_number":938,"context_line":"        # there won\u0027t be more than 2 expirer task cleanup async, see"},{"line_number":939,"context_line":"        # proxy.controller.obj.NUM_CLEAN_EXPIRER_QUEUE_UPDATES"},{"line_number":940,"context_line":"        self.assertLessEqual(len(async_updates), 2)"},{"line_number":941,"context_line":"        for up in async_updates:"},{"line_number":942,"context_line":"            self.assertEqual(\u0027DELETE\u0027, up[\u0027op\u0027])"},{"line_number":943,"context_line":"            self.assertEqual(\u0027.expiring_objects\u0027, up[\u0027account\u0027])"}],"source_content_type":"text/x-python","patch_set":2,"id":"9a9ed477_5fe4aee2","line":940,"updated":"2025-01-16 20:30:42.000000000","message":"Here I thought it was because we stopped 2/4 nodes...","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"b6fb8668d7a23afaa4284824efd3e2c261f3b900","unresolved":true,"context_lines":[{"line_number":937,"context_line":"                async_updates.append(pickle.load(f))"},{"line_number":938,"context_line":"        # there won\u0027t be more than 2 expirer task cleanup async, see"},{"line_number":939,"context_line":"        # proxy.controller.obj.NUM_CLEAN_EXPIRER_QUEUE_UPDATES"},{"line_number":940,"context_line":"        self.assertLessEqual(len(async_updates), 2)"},{"line_number":941,"context_line":"        for up in async_updates:"},{"line_number":942,"context_line":"            self.assertEqual(\u0027DELETE\u0027, up[\u0027op\u0027])"},{"line_number":943,"context_line":"            self.assertEqual(\u0027.expiring_objects\u0027, up[\u0027account\u0027])"}],"source_content_type":"text/x-python","patch_set":2,"id":"6edbb44d_95044129","line":940,"in_reply_to":"9a9ed477_5fe4aee2","updated":"2025-01-21 16:44:44.000000000","message":"yeah, happen to be number 2 too. but I guess this check is not the main target to test.","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ecf96f00a08fdf95c77b96314c7be61eef0727f4","unresolved":true,"context_lines":[{"line_number":963,"context_line":""},{"line_number":964,"context_line":"        # it\u0027s basically terrifying that if we don\u0027t run the object"},{"line_number":965,"context_line":"        # replicators here first, the stale entry might \"work\" on the"},{"line_number":966,"context_line":"        # handoff node, even tho the expirer-ic returns 409 (?)"},{"line_number":967,"context_line":"        Manager([\u0027object-replicator\u0027]).once()"},{"line_number":968,"context_line":""},{"line_number":969,"context_line":"        # object is expired, but not yet reaped"}],"source_content_type":"text/x-python","patch_set":2,"id":"8f2d6fb9_54f85c6f","line":966,"updated":"2025-01-16 20:30:42.000000000","message":"See https://bugs.launchpad.net/swift/+bug/1182628","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ecf96f00a08fdf95c77b96314c7be61eef0727f4","unresolved":true,"context_lines":[{"line_number":969,"context_line":"        # object is expired, but not yet reaped"},{"line_number":970,"context_line":"        self.assertObjectInListing()"},{"line_number":971,"context_line":"        # since we haven\u0027t run the object-updater both tasks are still in the"},{"line_number":972,"context_line":"        # queue, the one with orig_delete_at is \"stale\""},{"line_number":973,"context_line":"        expirer_tasks \u003d self.gather_expirer_tasks()"},{"line_number":974,"context_line":"        self.assertEqual(2, len(expirer_tasks))"},{"line_number":975,"context_line":"        # attempt to process both queue"}],"source_content_type":"text/x-python","patch_set":2,"id":"741c3c56_ecc975a0","line":972,"updated":"2025-01-16 20:30:42.000000000","message":"OK, but if the updater runs first, there\u0027s only one entry -- right?\n\nI was kinda imagining a test like\n\n- stop primary half\n- PUT at t1\n- start primary half\n- replicate on primary from handoff half (so all four nodes have the same .data)\n- stop handoff half\n- POST X-Delete-After: 1\n- start handoff half\n- stop primary half\n- POST X-Delete-After: 3600\n- start primary half\n- run updaters, replicators, etc -- basically `self.get_to_final_state()`\n- see that there\u0027s still two items in the queue\n- sleep, if needed (it probably won\u0027t be)\n- run expirer\n- see that there\u0027s now just one item in the queue\n\nBasically, force a properly stuck queue entry that previously would have required a PUT/DELETE to let it clear.","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"b6fb8668d7a23afaa4284824efd3e2c261f3b900","unresolved":true,"context_lines":[{"line_number":978,"context_line":"        # object is now reaped"},{"line_number":979,"context_line":"        self.assertObjectNotInListing()"},{"line_number":980,"context_line":"        # all queue tasks are popped"},{"line_number":981,"context_line":"        self.assertFalse(self.gather_expirer_tasks())"},{"line_number":982,"context_line":"        # async\u0027s have no effect"},{"line_number":983,"context_line":"        Manager([\u0027object-updater\u0027]).once()"},{"line_number":984,"context_line":"        self.assertFalse(self.gather_async_pendings())"}],"source_content_type":"text/x-python","patch_set":2,"id":"5ef3dd01_f2bc9806","line":981,"updated":"2025-01-21 16:44:44.000000000","message":"okay, this proves the stale delete task was removed as well, instead of keeping retrying until `reclaim_age`.\n\nI tried to change task_container_per_day option and restart expirers \u0026 object-servers in the middle of a probe test to create those stale delete tasks, but didn\u0027t find a way to do so. This probe test creates stale delete tasks by preventing object updater from running, TIL.","commit_id":"fc711c86ef6cb2825d96db1eb5b99a473a38e9d9"}]}