)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3c75b7a7a368d7c53948bd97cd684b148cc00126","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"298514ab_50f2b198","updated":"2024-10-01 04:20:12.000000000","message":"This patch introduces the rotation cycle for expirers to process different task containers, IIUC, this won\u0027t increase the parallelism of how many expirers will be processing those bloated containers at the same time.","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b2d02bedbdb42324090b386129edab31c431b57f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"e4ff549d_3981a964","updated":"2024-10-01 13:43:22.000000000","message":"it\u0027s just any idea; it\u0027s not the ONLY idea\n\n... but I think there\u0027s some advantages over just sticking with 100 containers per day forever.","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"}],"swift/obj/expirer.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bfcc08f6bae85285dee21fe0baea7b5f93ca5e66","unresolved":true,"context_lines":[{"line_number":585,"context_line":""},{"line_number":586,"context_line":"    def _calc_cycle_index(self):"},{"line_number":587,"context_line":"        try:"},{"line_number":588,"context_line":"            offset \u003d int(time() / self.interval % self.processes)"},{"line_number":589,"context_line":"        except ZeroDivisionError:"},{"line_number":590,"context_line":"            # when num_proccesses is 0 there is no offset"},{"line_number":591,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":1,"id":"884676fd_37e05545","line":588,"updated":"2024-09-30 22:43:38.000000000","message":"This is as same as below with new brackets per rules of python, but it this what did you mean?\n``offset \u003d int((time() / self.interval) % self.processes)``","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3c75b7a7a368d7c53948bd97cd684b148cc00126","unresolved":true,"context_lines":[{"line_number":645,"context_line":"        \"\"\""},{"line_number":646,"context_line":"        task_container_per_day \u003d self.expirer_config.task_container_per_day"},{"line_number":647,"context_line":"        my_containers \u003d []"},{"line_number":648,"context_line":"        cycle_index \u003d self._calc_cycle_index()"},{"line_number":649,"context_line":"        if self.processes \u003e\u003d task_container_per_day:"},{"line_number":650,"context_line":"            # each process maps to one task container per day, and handles a"},{"line_number":651,"context_line":"            # 1/divisor tasks in that container; each container maps to one or"}],"source_content_type":"text/x-python","patch_set":1,"id":"0fa8cc48_f441f597","line":648,"updated":"2024-10-01 04:20:12.000000000","message":"oic, ``cycle_index`` will be increased by 1 every interval. So for each new interval, all expirers are going to be assigned to new task containers. But for those 100 bloated old task containers, they will still be assigned to 100 new expirers only, that means at any moment, there will only 100 expirers working on those 100 task containers?","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b2d02bedbdb42324090b386129edab31c431b57f","unresolved":true,"context_lines":[{"line_number":645,"context_line":"        \"\"\""},{"line_number":646,"context_line":"        task_container_per_day \u003d self.expirer_config.task_container_per_day"},{"line_number":647,"context_line":"        my_containers \u003d []"},{"line_number":648,"context_line":"        cycle_index \u003d self._calc_cycle_index()"},{"line_number":649,"context_line":"        if self.processes \u003e\u003d task_container_per_day:"},{"line_number":650,"context_line":"            # each process maps to one task container per day, and handles a"},{"line_number":651,"context_line":"            # 1/divisor tasks in that container; each container maps to one or"}],"source_content_type":"text/x-python","patch_set":1,"id":"b54e1d54_cf334060","line":648,"in_reply_to":"0fa8cc48_f441f597","updated":"2024-10-01 13:43:22.000000000","message":"not if the first set of 100 expirers are still churning on them - then we\u0027d have 200 have one interval; 300 after two intervals; etc","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3c75b7a7a368d7c53948bd97cd684b148cc00126","unresolved":true,"context_lines":[{"line_number":687,"context_line":"            self.logger.increment(\u0027tasks.delayed\u0027)"},{"line_number":688,"context_line":"            return False"},{"line_number":689,"context_line":""},{"line_number":690,"context_line":"        if time() \u003e self.begin_cycle + self.interval:"},{"line_number":691,"context_line":"            # we\u0027ve been stuck running long enough, start skipping tasks"},{"line_number":692,"context_line":"            if self.hash_mod(\u0027%s/%s\u0027 % (task_container, task_object),"},{"line_number":693,"context_line":"                             self.processes) !\u003d self.process:"}],"source_content_type":"text/x-python","patch_set":1,"id":"59cfe575_b94cd009","line":690,"updated":"2024-10-01 04:20:12.000000000","message":"if it\u0027s going to skip rest of delete tasks, probably it will be better to skip them all on a high level, for example, within ``_process_container`` or even higher.","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b2d02bedbdb42324090b386129edab31c431b57f","unresolved":true,"context_lines":[{"line_number":687,"context_line":"            self.logger.increment(\u0027tasks.delayed\u0027)"},{"line_number":688,"context_line":"            return False"},{"line_number":689,"context_line":""},{"line_number":690,"context_line":"        if time() \u003e self.begin_cycle + self.interval:"},{"line_number":691,"context_line":"            # we\u0027ve been stuck running long enough, start skipping tasks"},{"line_number":692,"context_line":"            if self.hash_mod(\u0027%s/%s\u0027 % (task_container, task_object),"},{"line_number":693,"context_line":"                             self.processes) !\u003d self.process:"}],"source_content_type":"text/x-python","patch_set":1,"id":"36f9c319_fa72e7a2","line":690,"in_reply_to":"59cfe575_b94cd009","updated":"2024-10-01 13:43:22.000000000","message":"I agree it would ultimately be more optimal if we could somehow decide \"up front\" on the initial cycle after restart to assign the bloated containers to all expirers with skipping - essentially going back to the original \"task containers \u003c processes\" behavior... but it doesn\u0027t get us any closer to the future.\n\nThis way should help expirers when the encouter bloated containers to finish their cycle and get some new/differet containers assigned - reducing the time until we get to new/ready async/immedidate deletes while spreading the bloated work around the rotation.","commit_id":"a1fea093331c62ada553f210cd28582a8519533d"}]}