)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"3f8be5e8_9321f656","updated":"2023-05-03 18:30:34.000000000","message":"I like metrics!","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"aa86f456_0daf1f5b","updated":"2023-05-04 05:11:49.000000000","message":"Thanks very much for reviews!","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1b5a6d6ef59fd39a510af8da67d47391b6199d04","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"e2dbc0e8_367b83de","updated":"2023-05-10 06:43:57.000000000","message":"I\u0027ve pulled this into a SAIO and have a statsd + grafana container, and sure enough I see these timing metrics come out. Nice one Jianjian","commit_id":"744e9a94afea55ddc67cf8a1e8f580f8bf5927a4"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":true,"context_lines":[{"line_number":2334,"context_line":"        move_start_ts \u003d time.time()"},{"line_number":2335,"context_line":"        self._move_misplaced_objects(broker)"},{"line_number":2336,"context_line":"        self.logger.timing_since("},{"line_number":2337,"context_line":"            \u0027sharder.cleaving.move_misplaced\u0027, move_start_ts)"},{"line_number":2338,"context_line":""},{"line_number":2339,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self.auto_shard and not is_deleted"},{"line_number":2340,"context_line":"        if state in (UNSHARDED, COLLAPSED):"}],"source_content_type":"text/x-python","patch_set":1,"id":"744a5ef9_2416ef51","line":2337,"range":{"start_line":2337,"start_character":21,"end_line":2337,"end_character":29},"updated":"2023-05-03 18:30:34.000000000","message":"Maybe I\u0027m bike-shedding, but I\u0027m not sure about the naming here -- I think of cleaving and moving misplaced objects as *different* parts of processing databases.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":false,"context_lines":[{"line_number":2334,"context_line":"        move_start_ts \u003d time.time()"},{"line_number":2335,"context_line":"        self._move_misplaced_objects(broker)"},{"line_number":2336,"context_line":"        self.logger.timing_since("},{"line_number":2337,"context_line":"            \u0027sharder.cleaving.move_misplaced\u0027, move_start_ts)"},{"line_number":2338,"context_line":""},{"line_number":2339,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self.auto_shard and not is_deleted"},{"line_number":2340,"context_line":"        if state in (UNSHARDED, COLLAPSED):"}],"source_content_type":"text/x-python","patch_set":1,"id":"dcd5e01e_ce254582","line":2337,"range":{"start_line":2337,"start_character":21,"end_line":2337,"end_character":29},"in_reply_to":"744a5ef9_2416ef51","updated":"2023-05-04 05:11:49.000000000","message":"yes, they are. I remember we want to use \"cleaving\" to include both sharding and shrinking processes. But _process_broker is only sharding, let\u0027s use \"sharding\".","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":true,"context_lines":[{"line_number":2351,"context_line":"                    # container has been given shard ranges rather than"},{"line_number":2352,"context_line":"                    # found them e.g. via replication or a shrink event,"},{"line_number":2353,"context_line":"                    # or manually triggered cleaving."},{"line_number":2354,"context_line":"                    if broker.set_sharding_state():"},{"line_number":2355,"context_line":"                        state \u003d SHARDING"},{"line_number":2356,"context_line":"                        self.info(broker, \u0027Kick off container cleaving, \u0027"},{"line_number":2357,"context_line":"                                          \u0027own shard range in state %r\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1e24d297_a2ecda76","line":2354,"updated":"2023-05-03 18:30:34.000000000","message":"This might be another interesting spot to grab a metric -- if we\u0027ve enabled sharding, there\u0027s likely a lot of updates still coming in, so there may be some lock contention worth measuring.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":false,"context_lines":[{"line_number":2351,"context_line":"                    # container has been given shard ranges rather than"},{"line_number":2352,"context_line":"                    # found them e.g. via replication or a shrink event,"},{"line_number":2353,"context_line":"                    # or manually triggered cleaving."},{"line_number":2354,"context_line":"                    if broker.set_sharding_state():"},{"line_number":2355,"context_line":"                        state \u003d SHARDING"},{"line_number":2356,"context_line":"                        self.info(broker, \u0027Kick off container cleaving, \u0027"},{"line_number":2357,"context_line":"                                          \u0027own shard range in state %r\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"11c049ac_64988607","line":2354,"in_reply_to":"1e24d297_a2ecda76","updated":"2023-05-04 05:11:49.000000000","message":"Good point, we should measure it.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":true,"context_lines":[{"line_number":2411,"context_line":"                                            shard_range.container,"},{"line_number":2412,"context_line":"                                            [shard_range])"},{"line_number":2413,"context_line":"                self.logger.timing_since("},{"line_number":2414,"context_line":"                    \u0027sharder.cleaving.send_sr\u0027, send_start_ts)"},{"line_number":2415,"context_line":""},{"line_number":2416,"context_line":"            if not broker.is_root_container():"},{"line_number":2417,"context_line":"                # Update the root container with this container\u0027s shard range"}],"source_content_type":"text/x-python","patch_set":1,"id":"174df4e7_e00faab6","line":2414,"range":{"start_line":2414,"start_character":29,"end_line":2414,"end_character":37},"updated":"2023-05-03 18:30:34.000000000","message":"Again, not sure this has much to do with cleaving...","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":false,"context_lines":[{"line_number":2411,"context_line":"                                            shard_range.container,"},{"line_number":2412,"context_line":"                                            [shard_range])"},{"line_number":2413,"context_line":"                self.logger.timing_since("},{"line_number":2414,"context_line":"                    \u0027sharder.cleaving.send_sr\u0027, send_start_ts)"},{"line_number":2415,"context_line":""},{"line_number":2416,"context_line":"            if not broker.is_root_container():"},{"line_number":2417,"context_line":"                # Update the root container with this container\u0027s shard range"}],"source_content_type":"text/x-python","patch_set":1,"id":"8f42134f_15067282","line":2414,"range":{"start_line":2414,"start_character":29,"end_line":2414,"end_character":37},"in_reply_to":"174df4e7_e00faab6","updated":"2023-05-04 05:11:49.000000000","message":"Done","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":true,"context_lines":[{"line_number":2423,"context_line":"                update_start_ts \u003d time.time()"},{"line_number":2424,"context_line":"                self._update_root_container(broker)"},{"line_number":2425,"context_line":"                self.logger.timing_since("},{"line_number":2426,"context_line":"                    \u0027sharder.cleaving.update_root\u0027, update_start_ts)"},{"line_number":2427,"context_line":""},{"line_number":2428,"context_line":"        self.debug(broker,"},{"line_number":2429,"context_line":"                   \u0027Finished processing, state %s%s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"622d4888_e0d7df3e","line":2426,"range":{"start_line":2426,"start_character":29,"end_line":2426,"end_character":37},"updated":"2023-05-03 18:30:34.000000000","message":"Here, too.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d26eaf9440f2d765ee2d84fbb2299bb6813969f8","unresolved":true,"context_lines":[{"line_number":2423,"context_line":"                update_start_ts \u003d time.time()"},{"line_number":2424,"context_line":"                self._update_root_container(broker)"},{"line_number":2425,"context_line":"                self.logger.timing_since("},{"line_number":2426,"context_line":"                    \u0027sharder.cleaving.update_root\u0027, update_start_ts)"},{"line_number":2427,"context_line":""},{"line_number":2428,"context_line":"        self.debug(broker,"},{"line_number":2429,"context_line":"                   \u0027Finished processing, state %s%s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"3169b9c5_81937be3","line":2426,"updated":"2023-05-03 03:23:00.000000000","message":"need test case for this.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":false,"context_lines":[{"line_number":2423,"context_line":"                update_start_ts \u003d time.time()"},{"line_number":2424,"context_line":"                self._update_root_container(broker)"},{"line_number":2425,"context_line":"                self.logger.timing_since("},{"line_number":2426,"context_line":"                    \u0027sharder.cleaving.update_root\u0027, update_start_ts)"},{"line_number":2427,"context_line":""},{"line_number":2428,"context_line":"        self.debug(broker,"},{"line_number":2429,"context_line":"                   \u0027Finished processing, state %s%s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"34e9729a_57ae6c1f","line":2426,"range":{"start_line":2426,"start_character":29,"end_line":2426,"end_character":37},"in_reply_to":"622d4888_e0d7df3e","updated":"2023-05-04 05:11:49.000000000","message":"Done","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38de7d439c6bf934b37e0624a4d8d2b2af9de024","unresolved":true,"context_lines":[{"line_number":2493,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":2494,"context_line":"                else:"},{"line_number":2495,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027skipped\u0027)"},{"line_number":2496,"context_line":"            except (Exception, Timeout) as err:"},{"line_number":2497,"context_line":"                self._increment_stat(\u0027visited\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":2498,"context_line":"                self.exception(broker, \u0027Unhandled exception while processing: \u0027"},{"line_number":2499,"context_line":"                                       \u0027%s\u0027, err)"}],"source_content_type":"text/x-python","patch_set":1,"id":"605adc95_99989225","line":2496,"updated":"2023-05-03 18:30:34.000000000","message":"If there was a timeout in any of those operations, it\u0027ll bubble out to here, and we won\u0027t have timings -- was that intentional, or do we want those `timing_since` calls in `finally` blocks?","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1b5a6d6ef59fd39a510af8da67d47391b6199d04","unresolved":true,"context_lines":[{"line_number":2493,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":2494,"context_line":"                else:"},{"line_number":2495,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027skipped\u0027)"},{"line_number":2496,"context_line":"            except (Exception, Timeout) as err:"},{"line_number":2497,"context_line":"                self._increment_stat(\u0027visited\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":2498,"context_line":"                self.exception(broker, \u0027Unhandled exception while processing: \u0027"},{"line_number":2499,"context_line":"                                       \u0027%s\u0027, err)"}],"source_content_type":"text/x-python","patch_set":1,"id":"c25ae640_7a072028","line":2496,"in_reply_to":"3685690b_1d68a1d4","updated":"2023-05-10 06:43:57.000000000","message":"yeah maybe, but at least these metrics are better then what we had before, so let\u0027s not hold this up. We can already improve metrics in follow up patches when we need em.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"575906a388e1bba1bc0e391c75e625b855112db0","unresolved":true,"context_lines":[{"line_number":2493,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":2494,"context_line":"                else:"},{"line_number":2495,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027skipped\u0027)"},{"line_number":2496,"context_line":"            except (Exception, Timeout) as err:"},{"line_number":2497,"context_line":"                self._increment_stat(\u0027visited\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":2498,"context_line":"                self.exception(broker, \u0027Unhandled exception while processing: \u0027"},{"line_number":2499,"context_line":"                                       \u0027%s\u0027, err)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3685690b_1d68a1d4","line":2496,"in_reply_to":"605adc95_99989225","updated":"2023-05-04 05:11:49.000000000","message":"In the past and this week as well, we have seen some very slow sharding cycles (near 4 hours), that\u0027s where those new timing metrics can help for investigation. For potential timeouts or exceptions, I feel probably logging could be more helpful, we are logging total time of \"_one_shard_cycle\" already, more printed logs will be needed.","commit_id":"476ffacc5d74771257d5efcd56057b6bb7c68919"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1b5a6d6ef59fd39a510af8da67d47391b6199d04","unresolved":false,"context_lines":[{"line_number":2396,"context_line":"                                      \u0027state\u0027)"},{"line_number":2397,"context_line":"                    self.logger.timing_since("},{"line_number":2398,"context_line":"                        \u0027sharder.sharding.completed\u0027,"},{"line_number":2399,"context_line":"                        broker.get_own_shard_range().epoch)"},{"line_number":2400,"context_line":"                else:"},{"line_number":2401,"context_line":"                    self.info(broker, \u0027Completed cleaving, DB remaining in \u0027"},{"line_number":2402,"context_line":"                                      \u0027sharding state\u0027)"}],"source_content_type":"text/x-python","patch_set":2,"id":"b7d0344f_8113d923","line":2399,"updated":"2023-05-10 06:43:57.000000000","message":"Love this one! Great idea!","commit_id":"744e9a94afea55ddc67cf8a1e8f580f8bf5927a4"}]}