)]}'
{"swift/obj/diskfile.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7ebfcf8bc9e328c5b4ef2e1a9b806123d616bc08","unresolved":false,"context_lines":[{"line_number":1380,"context_line":"        async_dir \u003d os.path.join(device_path, get_async_dir(policy))"},{"line_number":1381,"context_line":"        tmp_dir \u003d os.path.join(device_path, get_tmp_dir(policy))"},{"line_number":1382,"context_line":"        mkdirs(tmp_dir)"},{"line_number":1383,"context_line":"        chash \u003d hash_path(account, container)"},{"line_number":1384,"context_line":"        ohash \u003d hash_path(account, container, obj)"},{"line_number":1385,"context_line":"        write_pickle("},{"line_number":1386,"context_line":"            data,"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f4c43b2_50510e62","line":1383,"updated":"2020-04-17 18:40:42.000000000","message":"How\u0027s this going to interact with sharded containers?","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":1380,"context_line":"        async_dir \u003d os.path.join(device_path, get_async_dir(policy))"},{"line_number":1381,"context_line":"        tmp_dir \u003d os.path.join(device_path, get_tmp_dir(policy))"},{"line_number":1382,"context_line":"        mkdirs(tmp_dir)"},{"line_number":1383,"context_line":"        chash \u003d hash_path(account, container)"},{"line_number":1384,"context_line":"        ohash \u003d hash_path(account, container, obj)"},{"line_number":1385,"context_line":"        write_pickle("},{"line_number":1386,"context_line":"            data,"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_2d4306b1","line":1383,"in_reply_to":"3f4c43b2_50510e62","updated":"2020-04-28 23:40:37.000000000","message":"The directory is now the hash of the container or container-shard. By segmenting more, we can be more efficient in handling container-server troubles","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"}],"swift/obj/updater.py":[{"author":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"change_message_id":"1fcb6a4e41dffe3bb5dd79254f58b2775f9cd5d1","unresolved":false,"context_lines":[{"line_number":206,"context_line":"        dump_recon_cache({\u0027object_updater_sweep\u0027: elapsed},"},{"line_number":207,"context_line":"                         self.rcache, self.logger)"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"    def _iter_async_pendings(self, device):"},{"line_number":210,"context_line":"        \"\"\""},{"line_number":211,"context_line":"        Locate and yield all the async pendings on the device. Multiple updates"},{"line_number":212,"context_line":"        for the same object will come out in reverse-chronological order"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_675b202e","line":209,"updated":"2020-04-10 21:31:28.000000000","message":"Method should be renamed","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f09c9d7ea56bd6effdbfbef054955e0207cb77a4","unresolved":false,"context_lines":[{"line_number":245,"context_line":"        else:"},{"line_number":246,"context_line":"            time_limit \u003d None"},{"line_number":247,"context_line":"        failures \u003d 0"},{"line_number":248,"context_line":"        for update in sorted(self._listdir(prefix_path), reverse\u003dTrue):"},{"line_number":249,"context_line":"            update_path \u003d os.path.join(prefix_path, update)"},{"line_number":250,"context_line":"            if not os.path.isfile(update_path):"},{"line_number":251,"context_line":"                continue"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_672680ba","line":248,"range":{"start_line":248,"start_character":34,"end_line":248,"end_character":42},"updated":"2020-04-10 21:28:37.000000000","message":"I\u0027m a little bit worried about the timing on this listdir -- previously we had updates splayed out by suffix, but now all updates for a container will be in the same directory... given that we\u0027re worried about a single container being overloaded or offline, leading to a pile-up of pendings... I\u0027m concerned about the prospect of having hundreds of thousands or even millions of updates slow the ls.\n\nDepending on what you configured, it could even get to be enough to dominate the time limit :-/\n\nMaybe the path should be more like \u003cc-hash\u003e/\u003co-suffix\u003e/\u003co-hash\u003e-\u003co-timestamp\u003e?","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":245,"context_line":"        else:"},{"line_number":246,"context_line":"            time_limit \u003d None"},{"line_number":247,"context_line":"        failures \u003d 0"},{"line_number":248,"context_line":"        for update in sorted(self._listdir(prefix_path), reverse\u003dTrue):"},{"line_number":249,"context_line":"            update_path \u003d os.path.join(prefix_path, update)"},{"line_number":250,"context_line":"            if not os.path.isfile(update_path):"},{"line_number":251,"context_line":"                continue"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_cdfd426f","line":248,"range":{"start_line":248,"start_character":34,"end_line":248,"end_character":42},"in_reply_to":"7f423b7e_672680ba","updated":"2020-04-28 23:40:37.000000000","message":"Right. The new patch create subdirectories under the container-directory, and it also avoid counting listdir and rate-limitation for the time spent on a container-directory.","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"change_message_id":"1fcb6a4e41dffe3bb5dd79254f58b2775f9cd5d1","unresolved":false,"context_lines":[{"line_number":266,"context_line":"            # If there are multiple updates for a given object,"},{"line_number":267,"context_line":"            # they\u0027ll look like this:"},{"line_number":268,"context_line":"            #"},{"line_number":269,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp1\u003e"},{"line_number":270,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp2\u003e"},{"line_number":271,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp3\u003e"},{"line_number":272,"context_line":"            #"},{"line_number":273,"context_line":"            # Async updates also have the property that newer"},{"line_number":274,"context_line":"            # updates contain all the information in older updates."}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_c73af4cb","line":271,"range":{"start_line":269,"start_character":0,"end_line":271,"end_character":73},"updated":"2020-04-10 21:31:28.000000000","message":"Comment should be updated (account/container hash)","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":266,"context_line":"            # If there are multiple updates for a given object,"},{"line_number":267,"context_line":"            # they\u0027ll look like this:"},{"line_number":268,"context_line":"            #"},{"line_number":269,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp1\u003e"},{"line_number":270,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp2\u003e"},{"line_number":271,"context_line":"            # \u003cdevice\u003e/async_pending/\u003cobj_suffix\u003e/\u003cobj_hash\u003e-\u003ctimestamp3\u003e"},{"line_number":272,"context_line":"            #"},{"line_number":273,"context_line":"            # Async updates also have the property that newer"},{"line_number":274,"context_line":"            # updates contain all the information in older updates."}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_4df13295","line":271,"range":{"start_line":269,"start_character":0,"end_line":271,"end_character":73},"in_reply_to":"7f423b7e_c73af4cb","updated":"2020-04-28 23:40:37.000000000","message":"Done","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f09c9d7ea56bd6effdbfbef054955e0207cb77a4","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            # are obsolete."},{"line_number":279,"context_line":"            #"},{"line_number":280,"context_line":"            # This way, our caller only gets useful async_pendings."},{"line_number":281,"context_line":"            if obj_hash \u003d\u003d last_obj_hash:"},{"line_number":282,"context_line":"                self.stats.unlinks +\u003d 1"},{"line_number":283,"context_line":"                self.logger.increment(\u0027unlinks\u0027)"},{"line_number":284,"context_line":"                os.unlink(update_path)"},{"line_number":285,"context_line":"            else:"},{"line_number":286,"context_line":"                last_obj_hash \u003d obj_hash"},{"line_number":287,"context_line":"                if not self.process_object_update(update_path, device, policy):"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_a73c08c0","line":284,"range":{"start_line":281,"start_character":12,"end_line":284,"end_character":38},"updated":"2020-04-10 21:28:37.000000000","message":"We should find a way to still do this even when the time limit\u0027s up. Worst case with this patch, we\u0027ve gone from\n\n obj1-t3    processed\n obj1-t2    deleted\n obj1-t1    deleted\n obj2-t4    processed\n\nto\n\n obj1-t3    processed\n            (time limit exceeded)\n obj1-t2    processed, but out of date\n            (time limit exceeded)\n obj1-t1    processed, but out of date\n            (time limit exceeded)\n obj2-t4    processed","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            # are obsolete."},{"line_number":279,"context_line":"            #"},{"line_number":280,"context_line":"            # This way, our caller only gets useful async_pendings."},{"line_number":281,"context_line":"            if obj_hash \u003d\u003d last_obj_hash:"},{"line_number":282,"context_line":"                self.stats.unlinks +\u003d 1"},{"line_number":283,"context_line":"                self.logger.increment(\u0027unlinks\u0027)"},{"line_number":284,"context_line":"                os.unlink(update_path)"},{"line_number":285,"context_line":"            else:"},{"line_number":286,"context_line":"                last_obj_hash \u003d obj_hash"},{"line_number":287,"context_line":"                if not self.process_object_update(update_path, device, policy):"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_2dee66b3","line":284,"range":{"start_line":281,"start_character":12,"end_line":284,"end_character":38},"in_reply_to":"7f423b7e_a73c08c0","updated":"2020-04-28 23:40:37.000000000","message":"Done","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f09c9d7ea56bd6effdbfbef054955e0207cb77a4","unresolved":false,"context_lines":[{"line_number":286,"context_line":"                last_obj_hash \u003d obj_hash"},{"line_number":287,"context_line":"                if not self.process_object_update(update_path, device, policy):"},{"line_number":288,"context_line":"                    failures +\u003d 1"},{"line_number":289,"context_line":"                if self.max_prefix_failures and \\"},{"line_number":290,"context_line":"                        failures \u003e\u003d self.max_prefix_failures:"},{"line_number":291,"context_line":"                    self.logger.error(\u0027Skipping prefix %s, too many errors\u0027"},{"line_number":292,"context_line":"                                      \u0027 (%d)\u0027 % (prefix_path, failures))"},{"line_number":293,"context_line":"                    break"},{"line_number":294,"context_line":"                if time_limit and time.time() \u003e\u003d time_limit:"},{"line_number":295,"context_line":"                    self.logger.error(\u0027Skipping prefix %s, time limit elapsed\u0027"},{"line_number":296,"context_line":"                                      \u0027 (%d)\u0027 % (prefix_path,"},{"line_number":297,"context_line":"                                                 self.max_prefix_time))"},{"line_number":298,"context_line":"                    break"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    def object_sweep(self, device):"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_e7461050","line":297,"range":{"start_line":289,"start_character":16,"end_line":297,"end_character":71},"updated":"2020-04-10 21:28:37.000000000","message":"Worth adding a stat for these, too?","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"change_message_id":"1fcb6a4e41dffe3bb5dd79254f58b2775f9cd5d1","unresolved":false,"context_lines":[{"line_number":296,"context_line":"                                      \u0027 (%d)\u0027 % (prefix_path,"},{"line_number":297,"context_line":"                                                 self.max_prefix_time))"},{"line_number":298,"context_line":"                    break"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    def object_sweep(self, device):"},{"line_number":301,"context_line":"        \"\"\""},{"line_number":302,"context_line":"        If there are async pendings on the device, walk each one and update."}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_876f4cc4","line":299,"updated":"2020-04-10 21:31:28.000000000","message":"Should be a good idea to delete container ohash folder if all update_path was unlinked. That was not a problem previously with ohash suffix (max 4096 folders). But with container that should grow a lot.","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":296,"context_line":"                                      \u0027 (%d)\u0027 % (prefix_path,"},{"line_number":297,"context_line":"                                                 self.max_prefix_time))"},{"line_number":298,"context_line":"                    break"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    def object_sweep(self, device):"},{"line_number":301,"context_line":"        \"\"\""},{"line_number":302,"context_line":"        If there are async pendings on the device, walk each one and update."}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_8d2cda02","line":299,"in_reply_to":"7f423b7e_876f4cc4","updated":"2020-04-28 23:40:37.000000000","message":"Done","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"change_message_id":"1fcb6a4e41dffe3bb5dd79254f58b2775f9cd5d1","unresolved":false,"context_lines":[{"line_number":310,"context_line":"        self.logger.info(\"Object update sweep starting on %s (pid: %d)\","},{"line_number":311,"context_line":"                         device, my_pid)"},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"        ap_iter \u003d RateLimitedIterator("},{"line_number":314,"context_line":"            self._iter_async_pendings(device),"},{"line_number":315,"context_line":"            elements_per_second\u003dself.max_objects_per_second)"},{"line_number":316,"context_line":"        with ContextPool(self.concurrency) as pool:"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f423b7e_0735fcda","line":313,"updated":"2020-04-10 21:31:28.000000000","message":"ap_iter should by renamed, it\u0027s not async pending dict anymore","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","unresolved":false,"context_lines":[{"line_number":310,"context_line":"        self.logger.info(\"Object update sweep starting on %s (pid: %d)\","},{"line_number":311,"context_line":"                         device, my_pid)"},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"        ap_iter \u003d RateLimitedIterator("},{"line_number":314,"context_line":"            self._iter_async_pendings(device),"},{"line_number":315,"context_line":"            elements_per_second\u003dself.max_objects_per_second)"},{"line_number":316,"context_line":"        with ContextPool(self.concurrency) as pool:"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_6d318e5a","line":313,"in_reply_to":"7f423b7e_0735fcda","updated":"2020-04-28 23:40:37.000000000","message":"Done","commit_id":"4e37d157d732ba37b1dfe50969f9345475f4b8ab"}]}