)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fb7a0612471705ba8814f89a32a12071eed99a93","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"da4f33d2_431cf5e3","updated":"2021-12-08 00:27:40.000000000","message":"recheck","commit_id":"bd6f2cdd901520a0fa259f4b0207faec9aa3618c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"a435da6d_0755b9df","updated":"2021-12-08 16:25:46.000000000","message":"it\u0027s really cool how easy you make it look to add error limiting to the updater!  Kudos on the refactor; it\u0027ll be interesting to play with and think about how to divide up the \"pools\" between proxies and other bg services that want to take advantage of knowledge sharing.","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"460e43e1daf9faff9fa49b72d484f8e27f8b341f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"3b6c1f67_5f635356","updated":"2022-09-22 17:24:57.000000000","message":"this is still pretty interesting\n\nI wonder if we\u0027re more trying to make the updater to faster by not trying to talk to error limited nodes - or if we imagine this error limiting helps the error limited nodes some how?\n\nOur goals are definately relevant, but one question I have is about tuning - do we have to go at least a whole cycle to get some benifit?  Do we ever want the updater workers to share state with each other (or the proxy!?) using memcache?","commit_id":"888b815de3859220d81383c0bd9c5208d8da7677"}],"swift/obj/updater.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fb7a0612471705ba8814f89a32a12071eed99a93","unresolved":true,"context_lines":[{"line_number":118,"context_line":"                                         DEFAULT_RECON_CACHE_PATH)"},{"line_number":119,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)"},{"line_number":120,"context_line":"        self.stats \u003d SweepStats()"},{"line_number":121,"context_line":"        # Call the mixin constructor to load up memcache client etc."},{"line_number":122,"context_line":"        self.error_limiter \u003d ErrorLimiter(conf, self.logger)"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"    def _listdir(self, path):"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa87598c_8bc76194","line":121,"range":{"start_line":121,"start_character":19,"end_line":121,"end_character":24},"updated":"2021-12-08 00:27:40.000000000","message":"ops need to update this comment.","commit_id":"bd6f2cdd901520a0fa259f4b0207faec9aa3618c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":385,"context_line":"            obj \u003d \u0027/%s/%s/%s\u0027 % (acct, cont, update[\u0027obj\u0027])"},{"line_number":386,"context_line":"            events \u003d [spawn(self.object_update,"},{"line_number":387,"context_line":"                            node, part, update[\u0027op\u0027], obj, headers_out)"},{"line_number":388,"context_line":"                      for node in nodes if node[\u0027id\u0027] not in successes]"},{"line_number":389,"context_line":"            success \u003d True"},{"line_number":390,"context_line":"            new_successes \u003d rewrite_pickle \u003d False"},{"line_number":391,"context_line":"            redirect \u003d None"}],"source_content_type":"text/x-python","patch_set":5,"id":"27bfed57_9653ed5e","line":388,"updated":"2021-12-08 16:25:46.000000000","message":"... and not is_error_limited(node)","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9afc952dbc8dea1dd5e55737d2c33fdd5fa447fc","unresolved":false,"context_lines":[{"line_number":385,"context_line":"            obj \u003d \u0027/%s/%s/%s\u0027 % (acct, cont, update[\u0027obj\u0027])"},{"line_number":386,"context_line":"            events \u003d [spawn(self.object_update,"},{"line_number":387,"context_line":"                            node, part, update[\u0027op\u0027], obj, headers_out)"},{"line_number":388,"context_line":"                      for node in nodes if node[\u0027id\u0027] not in successes]"},{"line_number":389,"context_line":"            success \u003d True"},{"line_number":390,"context_line":"            new_successes \u003d rewrite_pickle \u003d False"},{"line_number":391,"context_line":"            redirect \u003d None"}],"source_content_type":"text/x-python","patch_set":5,"id":"77fde817_3437a325","line":388,"in_reply_to":"27bfed57_9653ed5e","updated":"2021-12-09 04:27:29.000000000","message":"Done","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":444,"context_line":"                msg \u003d \u0027Update failed for %(obj)s %(path)s.\u0027"},{"line_number":445,"context_line":"                if error_limited_nodes:"},{"line_number":446,"context_line":"                    msg +\u003d (\" Error limited: %s\" %"},{"line_number":447,"context_line":"                            (\", \".join(error_limited_nodes)))"},{"line_number":448,"context_line":"                self.logger.debug(msg, {\u0027obj\u0027: obj, \u0027path\u0027: update_path})"},{"line_number":449,"context_line":"                if new_successes:"},{"line_number":450,"context_line":"                    update[\u0027successes\u0027] \u003d successes"}],"source_content_type":"text/x-python","patch_set":5,"id":"d07dbec1_772f796f","line":447,"updated":"2021-12-08 16:25:46.000000000","message":"i think we could ask this question of the error_limiter too\n\n    error_limited_nodes \u003d [node for node in nodes if self.error_limiter.is_error_limited(node)]","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9afc952dbc8dea1dd5e55737d2c33fdd5fa447fc","unresolved":true,"context_lines":[{"line_number":444,"context_line":"                msg \u003d \u0027Update failed for %(obj)s %(path)s.\u0027"},{"line_number":445,"context_line":"                if error_limited_nodes:"},{"line_number":446,"context_line":"                    msg +\u003d (\" Error limited: %s\" %"},{"line_number":447,"context_line":"                            (\", \".join(error_limited_nodes)))"},{"line_number":448,"context_line":"                self.logger.debug(msg, {\u0027obj\u0027: obj, \u0027path\u0027: update_path})"},{"line_number":449,"context_line":"                if new_successes:"},{"line_number":450,"context_line":"                    update[\u0027successes\u0027] \u003d successes"}],"source_content_type":"text/x-python","patch_set":5,"id":"b0ecd772_ec4f0601","line":447,"in_reply_to":"d07dbec1_772f796f","updated":"2021-12-09 04:27:29.000000000","message":"Rather then probing memcache for every node, just in case, I\u0027ll have a play with tracking recently discovered error limited nodes for the last supression time in the limter itself. let\u0027s see how that shakes out.","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":476,"context_line":"        \"\"\""},{"line_number":477,"context_line":"        redirect \u003d None"},{"line_number":478,"context_line":"        if self.error_limiter.error_limited(node):"},{"line_number":479,"context_line":"            node[\u0027error_limited\u0027] \u003d True"},{"line_number":480,"context_line":"            return False, node, redirect"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            with ConnectionTimeout(self.conn_timeout):"}],"source_content_type":"text/x-python","patch_set":5,"id":"656e1dfc_8a4a821d","line":479,"updated":"2021-12-08 16:25:46.000000000","message":"this is mostly to pass context to caller; I wonder if we could avoid spawning the event when the node is error limited","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9afc952dbc8dea1dd5e55737d2c33fdd5fa447fc","unresolved":false,"context_lines":[{"line_number":476,"context_line":"        \"\"\""},{"line_number":477,"context_line":"        redirect \u003d None"},{"line_number":478,"context_line":"        if self.error_limiter.error_limited(node):"},{"line_number":479,"context_line":"            node[\u0027error_limited\u0027] \u003d True"},{"line_number":480,"context_line":"            return False, node, redirect"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            with ConnectionTimeout(self.conn_timeout):"}],"source_content_type":"text/x-python","patch_set":5,"id":"10de3f69_b0b4b559","line":479,"in_reply_to":"656e1dfc_8a4a821d","updated":"2021-12-09 04:27:29.000000000","message":"Done","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"460e43e1daf9faff9fa49b72d484f8e27f8b341f","unresolved":true,"context_lines":[{"line_number":687,"context_line":"                if error_limited:"},{"line_number":688,"context_line":"                    msg +\u003d (\" Error limited: %s\" %"},{"line_number":689,"context_line":"                            (\", \".join(error_limited)))"},{"line_number":690,"context_line":"                self.logger.debug(msg, {\u0027obj\u0027: obj, \u0027path\u0027: update_path})"},{"line_number":691,"context_line":"                if new_successes:"},{"line_number":692,"context_line":"                    update[\u0027successes\u0027] \u003d successes"},{"line_number":693,"context_line":"                    rewrite_pickle \u003d True"}],"source_content_type":"text/x-python","patch_set":13,"id":"fd72d650_93084ec4","line":690,"updated":"2022-09-22 17:24:57.000000000","message":"ugh, I think we\u0027re going to need some more transparent telementry on what\u0027s going on with error limiting than \"just\" some debug log lines.  But I\u0027m not sure statsd is right either...","commit_id":"888b815de3859220d81383c0bd9c5208d8da7677"}],"test/unit/obj/test_updater.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fb7a0612471705ba8814f89a32a12071eed99a93","unresolved":true,"context_lines":[{"line_number":1259,"context_line":"            \u0027devices\u0027: self.devices_dir,"},{"line_number":1260,"context_line":"            \u0027mount_check\u0027: \u0027false\u0027,"},{"line_number":1261,"context_line":"            \u0027swift_dir\u0027: self.testdir,"},{"line_number":1262,"context_line":"            \u0027error_limited_cache_disabled\u0027: False,"},{"line_number":1263,"context_line":"        }"},{"line_number":1264,"context_line":"        with mock.patch(\u0027swift.common.memcached.MemcacheRing\u0027,"},{"line_number":1265,"context_line":"                        return_value\u003dmemcache):"}],"source_content_type":"text/x-python","patch_set":4,"id":"0991a4da_ce8ade45","line":1262,"updated":"2021-12-08 00:27:40.000000000","message":"Looking back at this now, I wonder if the config option should be \u0027error_limited_cache_enabled\u0027 and defaulted to False. Rather then what feels like a double negative to enable.","commit_id":"bd6f2cdd901520a0fa259f4b0207faec9aa3618c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":1247,"context_line":"        debug_lines \u003d self.logger.get_lines_for_level(\u0027debug\u0027)"},{"line_number":1248,"context_line":"        self.assertIn(\u0027Node error limited 127.0.0.1:1 (sda1)\u0027, debug_lines)"},{"line_number":1249,"context_line":"        self.assertIn(\u0027Update failed for /a/c/o \u0027, debug_lines[1])"},{"line_number":1250,"context_line":"        self.assertIn(\u0027Error limited: 127.0.0.1:1/sda1\u0027, debug_lines[1])"},{"line_number":1251,"context_line":""},{"line_number":1252,"context_line":"    def test_obj_update_error_limited_with_memcache(self):"},{"line_number":1253,"context_line":"        memcache \u003d FakeMemcache()"}],"source_content_type":"text/x-python","patch_set":5,"id":"cd558240_8a4b0aa5","line":1250,"updated":"2021-12-08 16:25:46.000000000","message":"maybe open it up and verify the successes are in there?","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":1250,"context_line":"        self.assertIn(\u0027Error limited: 127.0.0.1:1/sda1\u0027, debug_lines[1])"},{"line_number":1251,"context_line":""},{"line_number":1252,"context_line":"    def test_obj_update_error_limited_with_memcache(self):"},{"line_number":1253,"context_line":"        memcache \u003d FakeMemcache()"},{"line_number":1254,"context_line":"        policies \u003d list(POLICIES)"},{"line_number":1255,"context_line":"        random.shuffle(policies)"},{"line_number":1256,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"ea3bb3a5_99012e15","line":1253,"updated":"2021-12-08 16:25:46.000000000","message":"seems reasonable","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":1393,"context_line":"            daemon.error_limiter._error_limiting[\u0027/127.0.0.1:1/sda1\u0027])"},{"line_number":1394,"context_line":"        debug_lines \u003d self.logger.get_lines_for_level(\u0027debug\u0027)"},{"line_number":1395,"context_line":"        self.assertIn(\u0027Update failed for /a/c/o \u0027, debug_lines[-1])"},{"line_number":1396,"context_line":"        self.assertNotIn(\u0027Error limited: 127.0.0.1:1/sda1\u0027, debug_lines[-1])"},{"line_number":1397,"context_line":""},{"line_number":1398,"context_line":"        # So one more time and we\u0027ll see it\u0027s error limited"},{"line_number":1399,"context_line":"        self.logger.clear()"}],"source_content_type":"text/x-python","patch_set":5,"id":"ea94423e_a8fc8b7f","line":1396,"updated":"2021-12-08 16:25:46.000000000","message":"really there shouldn\u0027t be ANY \"Error limited\" lines...","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3fe7198d0d93ee833ccf771217f580179fbf1349","unresolved":true,"context_lines":[{"line_number":1486,"context_line":"                         memcache.get(node_key))"},{"line_number":1487,"context_line":"        debug_lines \u003d self.logger.get_lines_for_level(\u0027debug\u0027)"},{"line_number":1488,"context_line":"        self.assertIn(\u0027Update failed for /a/c/o \u0027, debug_lines[-1])"},{"line_number":1489,"context_line":"        self.assertIn(\u0027Error limited: 127.0.0.1:1/sda1\u0027, debug_lines[-1])"},{"line_number":1490,"context_line":""},{"line_number":1491,"context_line":""},{"line_number":1492,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":5,"id":"6178b471_e2d6f91c","line":1489,"updated":"2021-12-08 16:25:46.000000000","message":"so if we ran again right now this async wouldn\u0027t generate ANY requests right?\n\n.. and then maybe \"wait\" for the node to \"fall out\" of error limiting and then cleans up the async_pending?","commit_id":"ad32c0ec007f3383dcc0eae020d3accd47c06ad1"}]}