)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7d22df702f7077ec1eb5de23e249705f2e7f35cf","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Matthew Oliver \u003cmatt@oliver.net.au\u003e"},{"line_number":5,"context_line":"CommitDate: 2023-01-30 17:36:40 +1100"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Shrinking: add states and biggest root recon metrics"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When looking at shrinking candidates it might be nice to know how many"},{"line_number":10,"context_line":"active ranges there are to compare against the number of compactable"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"10c57dcf_ee35df1d","line":7,"updated":"2023-01-30 18:18:47.000000000","message":"so this is only recon, no statsd (anymore?)","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7d22df702f7077ec1eb5de23e249705f2e7f35cf","unresolved":true,"context_lines":[{"line_number":15,"context_line":"This patch adds shrinking to the state_count used by the"},{"line_number":16,"context_line":"sharding_inprogress code, but also includes the state_count in the"},{"line_number":17,"context_line":"shrinking candidates, shrinking candidates are root containers and"},{"line_number":18,"context_line":"knowing how many ranges are in what state is interesting."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"To have a rough idea of how big our biggest root containers are getting,"},{"line_number":21,"context_line":"now collect this and report the biggest (most active shards) for a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"996a3f3e_e3170b44","line":18,"updated":"2023-01-30 18:18:47.000000000","message":"the states might be interesting - I guess compactible_ranges already tells us most of what we need to know.","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Shrinking: add states and biggest_candidate root recon metrics"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When looking at shrinking candidates it might be nice to know how many"},{"line_number":10,"context_line":"active ranges there are to compare against the number of compactable"},{"line_number":11,"context_line":"ranges. Further when tracking the progress of cleaving (cleaving or"},{"line_number":12,"context_line":"shrinking) knowing how many ranges are in the shrinking state would also"},{"line_number":13,"context_line":"be nice to track."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"e6b9ff4c_eda5459a","line":11,"range":{"start_line":9,"start_character":62,"end_line":11,"end_character":6},"updated":"2023-02-17 15:36:29.000000000","message":"so, is the idea that we already know there are X compactable ranges, but now we also find out that there are Y total ranges?","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":9,"context_line":"When looking at shrinking candidates it might be nice to know how many"},{"line_number":10,"context_line":"active ranges there are to compare against the number of compactable"},{"line_number":11,"context_line":"ranges. Further when tracking the progress of cleaving (cleaving or"},{"line_number":12,"context_line":"shrinking) knowing how many ranges are in the shrinking state would also"},{"line_number":13,"context_line":"be nice to track."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This patch adds shrinking to the state_count used by the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"ff9d8582_edb8f781","line":12,"range":{"start_line":12,"start_character":19,"end_line":12,"end_character":62},"updated":"2023-02-17 15:36:29.000000000","message":"do we expect there to be any shrinking sub-shards of a shard that is cleaving? I didn\u0027t think we did, at least not in typical circumstances","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"829c057b434e4aa43d792bbf3e48e5c35700299f","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"bcef26ec_7b499a48","updated":"2023-01-06 06:14:10.000000000","message":"I am thinking, if we are going to monitor which container has potential shard ranges for shrinking, should the sharder calculate the average number of objects per shard for each container? let\u0027s see if one container has 1M objects and 1000 shards, then it\u0027s 1K objects per shard. And check this number against a threshold, if it\u0027s too low then we fire a metric or warning.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"984695466ba048cdc8f6fe2fc63647452c622b88","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"1384208a_44449e98","updated":"2023-01-30 05:34:56.000000000","message":"got a new patch incoming","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"72472d8d288733c5716143e383c922e2b9265136","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"e05b4df3_29496c43","updated":"2023-01-06 14:41:23.000000000","message":"i don\u0027t think these are the right metrics for the release monday","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"d65a7901_6f0f73e4","in_reply_to":"bcef26ec_7b499a48","updated":"2023-01-06 11:05:49.000000000","message":"In principle I agree - it would be good to have a metric that directly reports the potential for shrinking. But I think we already have the numbers available (not reported) in _identify_shrinking_candidates.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"685702617a14bb8761164a9adb81cfe70170599a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"3e7e8933_33760f86","updated":"2023-02-03 06:20:37.000000000","message":"I have a new patch incoming.","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7d22df702f7077ec1eb5de23e249705f2e7f35cf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"338d05f9_cf13accd","updated":"2023-01-30 18:18:47.000000000","message":"i\u0027m can see this change is tracking and collecting some more information, but I\u0027m not sure it\u0027s delivering the health metrics we\u0027re looking for - maybe we want to collect more info so we can tell what healthy looks like?","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"94a2fa7a_64519ff2","updated":"2023-02-17 15:36:29.000000000","message":"IMHO the more interesting of the two additions is the change to shrinking_candidates so that the total number of shards can be inferred as well as the number of compactible shards. So before we could see that there are N compactible shards, but now we can see there are N compactible out of a total of M shards.\n\nKnowing N/M is arguably more interesting than just N.\n\nRe. the top-N by number of shard ranges, I\u0027m not sure YET how useful the top-N is versus just to the top-1. What does an op do with the information? I might just need educating/convincing 😊","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f612a2d951d672159e06313d41acecc456142c4e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"27079f39_41e94266","updated":"2024-07-22 07:06:25.000000000","message":"I think this is something we can scrap. The biggest containers are not a bad thing and may never change.. so a pointless (as others pointed out).\n\nI\u0027m working on a simpler one that just adds better in progress support, but not major. We can also stop carrying this down stream.","commit_id":"68cc36f00b78c0ea62a2a5f4fc6adcc2baabaac6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"d0e299de_14676ba6","updated":"2023-03-10 02:22:40.000000000","message":"Thanks guys, clearly I might need to rethink this a little.\n\nAl, the cadidate thing might sound right because they aren\u0027t candidates for anything (or maybe they are, for looking into how big or if we need to increase the memcache max item size). But this way it just matches existing candiate behaviour. Have top x rather then top 1 means we can see if we have an outlier.","commit_id":"68cc36f00b78c0ea62a2a5f4fc6adcc2baabaac6"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[{"line_number":1266,"context_line":"        # of shard ranges and increment a value for each thousand range its in"},{"line_number":1267,"context_line":"        if range_count:"},{"line_number":1268,"context_line":"            range_len_thou_ceil \u003d ceil(range_count / 1000.0)"},{"line_number":1269,"context_line":"            self.logger.increment(\u0027audit_root.range_count.%d000_%d000\u0027 % ("},{"line_number":1270,"context_line":"                range_len_thou_ceil - 1, range_len_thou_ceil))"},{"line_number":1271,"context_line":""},{"line_number":1272,"context_line":"        # We\u0027ve seen a case in production where the roots own_shard_range"}],"source_content_type":"text/x-python","patch_set":2,"id":"540ac92a_3c63d379","line":1269,"range":{"start_line":1269,"start_character":17,"end_line":1269,"end_character":33},"updated":"2023-01-06 11:05:49.000000000","message":"AFAICT all other stats are reported via recon dumps. That may not be *great* but is it worth remaining consistent and adding this stat the recon dump?","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":false,"context_lines":[{"line_number":1266,"context_line":"        # of shard ranges and increment a value for each thousand range its in"},{"line_number":1267,"context_line":"        if range_count:"},{"line_number":1268,"context_line":"            range_len_thou_ceil \u003d ceil(range_count / 1000.0)"},{"line_number":1269,"context_line":"            self.logger.increment(\u0027audit_root.range_count.%d000_%d000\u0027 % ("},{"line_number":1270,"context_line":"                range_len_thou_ceil - 1, range_len_thou_ceil))"},{"line_number":1271,"context_line":""},{"line_number":1272,"context_line":"        # We\u0027ve seen a case in production where the roots own_shard_range"}],"source_content_type":"text/x-python","patch_set":2,"id":"37818a5a_dc5d511e","line":1269,"range":{"start_line":1269,"start_character":17,"end_line":1269,"end_character":33},"in_reply_to":"540ac92a_3c63d379","updated":"2023-02-17 15:36:29.000000000","message":"Done","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[{"line_number":1267,"context_line":"        if range_count:"},{"line_number":1268,"context_line":"            range_len_thou_ceil \u003d ceil(range_count / 1000.0)"},{"line_number":1269,"context_line":"            self.logger.increment(\u0027audit_root.range_count.%d000_%d000\u0027 % ("},{"line_number":1270,"context_line":"                range_len_thou_ceil - 1, range_len_thou_ceil))"},{"line_number":1271,"context_line":""},{"line_number":1272,"context_line":"        # We\u0027ve seen a case in production where the roots own_shard_range"},{"line_number":1273,"context_line":"        # epoch is reset to None, and state set to ACTIVE (like re-defaulted)"}],"source_content_type":"text/x-python","patch_set":2,"id":"77098f52_577af196","line":1270,"updated":"2023-01-06 11:05:49.000000000","message":"This metric would give us a picture of how big sharded containers are. I\u0027d suggest using some pseudo-log scale to get better granularity e.g. 0_10, 10_50, 50_100, 100_500, 500_1000, 1000_2000 etc\n\nBut having a large number of shard ranges is not in itself \u0027unhealthy\u0027. Unlike the container listing stat, where you\u0027d like to *never* see a count of more than say 2 or 3, a single instance of this metric tells us nothing about \u0027health\u0027 in terms of shrinkable shards. To be useful in that respect we\u0027d have to catch a delta, and even then I\u0027m not sure how insightful it would be e.g. \u0027range_count.7000_8000 has dropped from 9 to 8 on Monday (some shrinking happened) but then went from 8 to 9 on Tuesday (some sharding happened)\u0027.\n\nSince we already _identify_shrinking_candidates with a count of the number of compactible_ranges, would that be a better metric to report? In fact, would it be possible to report total number of (active) shards and total number of compactible shards? Those are the exact numbers we\u0027d like to see change when executing shrinking.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e0a4084a936d96953fe45c39c8bd58b5e5ce27a","unresolved":true,"context_lines":[{"line_number":1267,"context_line":"        if range_count:"},{"line_number":1268,"context_line":"            range_len_thou_ceil \u003d ceil(range_count / 1000.0)"},{"line_number":1269,"context_line":"            self.logger.increment(\u0027audit_root.range_count.%d000_%d000\u0027 % ("},{"line_number":1270,"context_line":"                range_len_thou_ceil - 1, range_len_thou_ceil))"},{"line_number":1271,"context_line":""},{"line_number":1272,"context_line":"        # We\u0027ve seen a case in production where the roots own_shard_range"},{"line_number":1273,"context_line":"        # epoch is reset to None, and state set to ACTIVE (like re-defaulted)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7a64d448_4e825be1","line":1270,"in_reply_to":"34ced279_d1f0f3e1","updated":"2023-01-30 02:47:00.000000000","message":"Yeah, OK maybe it isn\u0027t so useful. The problem I find is if we throw a metric we can only add it up, unless we have some more info, which is why I tried to break it up into buckets. This will be where next gen multi-label metrics could get better, we could add the account and container. \n\nI assume we\u0027d always have shards, but just wanted to get an idea of how big they were getting.\nBut I guess that\u0027s also out of scope for now or at least trying to do it as a metric probably doesn\u0027t make sense with what we\u0027ve currently got.\n\nMaybe for now, throwing the active number into the recon with the compactible would make it better so we can see not just a downward trend when shrinking is working, but also gives us an idea of scale.\n\nAnother recon metric could be max ACTIVE shard count so we get an idea of how big things are getting, if we ever want to track it across the cluster and get bigger contianers.\n\nTLDR, I\u0027ll pull away from throwning into statsd undil we have a better metric solution and throw into recon.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"72472d8d288733c5716143e383c922e2b9265136","unresolved":true,"context_lines":[{"line_number":1267,"context_line":"        if range_count:"},{"line_number":1268,"context_line":"            range_len_thou_ceil \u003d ceil(range_count / 1000.0)"},{"line_number":1269,"context_line":"            self.logger.increment(\u0027audit_root.range_count.%d000_%d000\u0027 % ("},{"line_number":1270,"context_line":"                range_len_thou_ceil - 1, range_len_thou_ceil))"},{"line_number":1271,"context_line":""},{"line_number":1272,"context_line":"        # We\u0027ve seen a case in production where the roots own_shard_range"},{"line_number":1273,"context_line":"        # epoch is reset to None, and state set to ACTIVE (like re-defaulted)"}],"source_content_type":"text/x-python","patch_set":2,"id":"34ced279_d1f0f3e1","line":1270,"in_reply_to":"77098f52_577af196","updated":"2023-01-06 14:41:23.000000000","message":"i\u0027m worried about the number of new metrics created here\n\neach new metric allocates a slab to store the time series data - as soon as carbon sees the first instance of a metric it creates a new ~400MB-4GB sparse file... we have billions of objects in some containers, 1B/1K ~\u003d 1M","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7d22df702f7077ec1eb5de23e249705f2e7f35cf","unresolved":true,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # finaly biggest"},{"line_number":1093,"context_line":"        self.stats[\u0027sharding\u0027][\u0027biggest\u0027] \u003d self.biggest_root"},{"line_number":1094,"context_line":""},{"line_number":1095,"context_line":"        dump_recon_cache("},{"line_number":1096,"context_line":"            {\u0027sharding_stats\u0027: self.stats,"}],"source_content_type":"text/x-python","patch_set":3,"id":"0d55e6be_2d010394","line":1093,"updated":"2023-01-30 18:18:47.000000000","message":"this might make more sense in recon as a top_n list like we do for shrinkgin/sharding candidates","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"685702617a14bb8761164a9adb81cfe70170599a","unresolved":true,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # finaly biggest"},{"line_number":1093,"context_line":"        self.stats[\u0027sharding\u0027][\u0027biggest\u0027] \u003d self.biggest_root"},{"line_number":1094,"context_line":""},{"line_number":1095,"context_line":"        dump_recon_cache("},{"line_number":1096,"context_line":"            {\u0027sharding_stats\u0027: self.stats,"}],"source_content_type":"text/x-python","patch_set":3,"id":"cb2674b7_9ee0be9b","line":1093,"in_reply_to":"0d55e6be_2d010394","updated":"2023-02-03 06:20:37.000000000","message":"yeah, great idea, top biggest makes sense.","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # finaly biggest"},{"line_number":1093,"context_line":"        self.stats[\u0027sharding\u0027][\u0027biggest\u0027] \u003d self.biggest_root"},{"line_number":1094,"context_line":""},{"line_number":1095,"context_line":"        dump_recon_cache("},{"line_number":1096,"context_line":"            {\u0027sharding_stats\u0027: self.stats,"}],"source_content_type":"text/x-python","patch_set":3,"id":"eb7c428d_297247da","line":1093,"in_reply_to":"cb2674b7_9ee0be9b","updated":"2023-02-17 15:36:29.000000000","message":"IDK, once we know the biggest has say 5000 ranges, what do we do with the info that the second biggest has 4500? And we\u0027d still have no info about the rest beyond top N?\n\nIs there an alert that an op would want to trigger, other than \u0027the biggest has more than \u003cthreshold\u003e shard ranges\u0027?","commit_id":"0868d3970070f015cf1b49c3bf151fdbbdf24a4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":916,"context_line":"        self.stats[\u0027sharding\u0027] \u003d defaultdict(lambda: defaultdict(int))"},{"line_number":917,"context_line":"        self.sharding_candidates \u003d []"},{"line_number":918,"context_line":"        self.shrinking_candidates \u003d []"},{"line_number":919,"context_line":"        self.big_candidates \u003d []"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"    def _append_stat(self, category, key, value):"},{"line_number":922,"context_line":"        if not self.stats[\u0027sharding\u0027][category][key]:"}],"source_content_type":"text/x-python","patch_set":4,"id":"c02c1aef_be0fcaf4","line":919,"updated":"2023-02-17 15:36:29.000000000","message":"I found the name confusing - \u0027big\u0027 immediately made me think \u0027object count\u0027, but actually it\u0027s \u0027shard count\u0027, and \u0027candidates\u0027 implies they are candidates for some action (like shrink or shard).","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":986,"context_line":"            # tool is used with the current max_shrinking, max_expanding"},{"line_number":987,"context_line":"            # settings."},{"line_number":988,"context_line":"            shrink_candidate[\u0027compactible_ranges\u0027] \u003d compactible_ranges"},{"line_number":989,"context_line":"            shrink_candidate.update(self._get_state_count(broker))"},{"line_number":990,"context_line":"            self.shrinking_candidates.append(shrink_candidate)"},{"line_number":991,"context_line":""},{"line_number":992,"context_line":"    def _transform_candidate_stats(self, category, candidates, sort_keys):"}],"source_content_type":"text/x-python","patch_set":4,"id":"498f21f9_ca8aea06","line":989,"updated":"2023-02-17 15:36:29.000000000","message":"ok, I can see how we would want to include shrinking state here, but then why not also shrunk, sharding, sharded etc\n\nperhaps _get_state_count needs to accept the set of states to include in the returned dict?","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":true,"context_lines":[{"line_number":986,"context_line":"            # tool is used with the current max_shrinking, max_expanding"},{"line_number":987,"context_line":"            # settings."},{"line_number":988,"context_line":"            shrink_candidate[\u0027compactible_ranges\u0027] \u003d compactible_ranges"},{"line_number":989,"context_line":"            shrink_candidate.update(self._get_state_count(broker))"},{"line_number":990,"context_line":"            self.shrinking_candidates.append(shrink_candidate)"},{"line_number":991,"context_line":""},{"line_number":992,"context_line":"    def _transform_candidate_stats(self, category, candidates, sort_keys):"}],"source_content_type":"text/x-python","patch_set":4,"id":"2221ae0e_50d6616d","line":989,"in_reply_to":"498f21f9_ca8aea06","updated":"2023-03-10 02:22:40.000000000","message":"yeah, I wanted to be able to track the number of shrinking ranges so we could montor progress of shrinking. And we already had code that counts and categorises sharding states so why not reuse it.\n\nI guess it doesn\u0027t hurt to categorise by all the shard states, but because it\u0027s shared code it would make the cleaving progress code contain all the states, and adding shrinking might be useful when we have shrinking enabled, but not expected in sharding.\nBut reconning all stat counts could be useful as we might enlighten us to other issues.","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":1000,"context_line":"    def _get_state_count(self, broker):"},{"line_number":1001,"context_line":"        states \u003d [ShardRange.FOUND, ShardRange.CREATED,"},{"line_number":1002,"context_line":"                  ShardRange.CLEAVED, ShardRange.ACTIVE,"},{"line_number":1003,"context_line":"                  ShardRange.SHRINKING]"},{"line_number":1004,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(states\u003dstates)"},{"line_number":1005,"context_line":"        state_count \u003d {}"},{"line_number":1006,"context_line":"        for state in states:"}],"source_content_type":"text/x-python","patch_set":4,"id":"89d38103_cdbffc0e","line":1003,"updated":"2023-02-17 15:36:29.000000000","message":"you\u0027ve added SHRINKING, but we don\u0027t expect a sub-shard to be SHRINKING while the parent is sharding?\n\nThe unit tests only ever assert shrinking \u003d 0\n\nAlso, I think this broke the probe test","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":true,"context_lines":[{"line_number":1000,"context_line":"    def _get_state_count(self, broker):"},{"line_number":1001,"context_line":"        states \u003d [ShardRange.FOUND, ShardRange.CREATED,"},{"line_number":1002,"context_line":"                  ShardRange.CLEAVED, ShardRange.ACTIVE,"},{"line_number":1003,"context_line":"                  ShardRange.SHRINKING]"},{"line_number":1004,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(states\u003dstates)"},{"line_number":1005,"context_line":"        state_count \u003d {}"},{"line_number":1006,"context_line":"        for state in states:"}],"source_content_type":"text/x-python","patch_set":4,"id":"f63bbee5_768e22ac","line":1003,"in_reply_to":"89d38103_cdbffc0e","updated":"2023-03-10 02:22:40.000000000","message":"no that\u0027s true, but when we do track shrinking candidates in the root, we\u0027d use the same code to categorize.","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"da4c7685a06d8ffbea1448dca9ba86af3ec11e8d","unresolved":true,"context_lines":[{"line_number":1032,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1033,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1034,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"},{"line_number":1035,"context_line":"        state_count \u003d self._get_state_count(broker)"},{"line_number":1036,"context_line":"        info.update(state_count)"},{"line_number":1037,"context_line":"        info[\u0027error\u0027] \u003d error and str(error)"},{"line_number":1038,"context_line":"        self._append_stat(\u0027sharding_in_progress\u0027, \u0027all\u0027, info)"}],"source_content_type":"text/x-python","patch_set":4,"id":"d0ae69a7_3cfbd1cb","line":1035,"updated":"2023-02-21 21:31:25.000000000","message":"So this now picks up `SHRINKING`... I think that\u0027s what\u0027s causing the probe test failures.","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":true,"context_lines":[{"line_number":1032,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1033,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1034,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"},{"line_number":1035,"context_line":"        state_count \u003d self._get_state_count(broker)"},{"line_number":1036,"context_line":"        info.update(state_count)"},{"line_number":1037,"context_line":"        info[\u0027error\u0027] \u003d error and str(error)"},{"line_number":1038,"context_line":"        self._append_stat(\u0027sharding_in_progress\u0027, \u0027all\u0027, info)"}],"source_content_type":"text/x-python","patch_set":4,"id":"6c99a8de_566ae964","line":1035,"in_reply_to":"d0ae69a7_3cfbd1cb","updated":"2023-03-10 02:22:40.000000000","message":"ahh yeah, I should make sure I fix probe tests.. and maybe run them sometimes :P","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"da4c7685a06d8ffbea1448dca9ba86af3ec11e8d","unresolved":true,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # And now track the biggest"},{"line_number":1093,"context_line":"        category \u003d self.stats[\u0027sharding\u0027][\u0027biggest_candidates\u0027]"},{"line_number":1094,"context_line":"        self._transform_candidate_stats(category, self.big_candidates,"},{"line_number":1095,"context_line":"                                        sort_keys\u003d(\u0027active\u0027,))"},{"line_number":1096,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"717f9841_4613365f","line":1093,"range":{"start_line":1093,"start_character":43,"end_line":1093,"end_character":61},"updated":"2023-02-21 21:31:25.000000000","message":"I never remember how recon works -- if we start carrying this and want to rename later, do we need to worry about the old name not getting cleaned up? Just trying to judge how much weight to give Alistair\u0027s concern at L919...","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"aa132ef95a6d09f4b80e366e6c8098ccd81402a4","unresolved":true,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # And now track the biggest"},{"line_number":1093,"context_line":"        category \u003d self.stats[\u0027sharding\u0027][\u0027biggest_candidates\u0027]"},{"line_number":1094,"context_line":"        self._transform_candidate_stats(category, self.big_candidates,"},{"line_number":1095,"context_line":"                                        sort_keys\u003d(\u0027active\u0027,))"},{"line_number":1096,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"bc595cb8_8afd7fae","line":1093,"range":{"start_line":1093,"start_character":43,"end_line":1093,"end_character":61},"in_reply_to":"717f9841_4613365f","updated":"2023-02-21 21:47:25.000000000","message":"Never mind -- just tried it out, any rename (or removal!) will automatically clean out the old entry.","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":false,"context_lines":[{"line_number":1090,"context_line":"                                        sort_keys\u003d(\u0027compactible_ranges\u0027,))"},{"line_number":1091,"context_line":""},{"line_number":1092,"context_line":"        # And now track the biggest"},{"line_number":1093,"context_line":"        category \u003d self.stats[\u0027sharding\u0027][\u0027biggest_candidates\u0027]"},{"line_number":1094,"context_line":"        self._transform_candidate_stats(category, self.big_candidates,"},{"line_number":1095,"context_line":"                                        sort_keys\u003d(\u0027active\u0027,))"},{"line_number":1096,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"316f692b_b19b5a0c","line":1093,"range":{"start_line":1093,"start_character":43,"end_line":1093,"end_character":61},"in_reply_to":"bc595cb8_8afd7fae","updated":"2023-03-10 02:22:40.000000000","message":"Ack","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":1297,"context_line":"            # Collect the biggest root container on a cycle"},{"line_number":1298,"context_line":"            if state \u003d\u003d ShardRange.ACTIVE:"},{"line_number":1299,"context_line":"                if len(self.big_candidates) \u003e\u003d self.recon_candidates_limit:"},{"line_number":1300,"context_line":"                    if self.big_candidates[-1][\u0027active\u0027] \u003c len(shard_ranges):"},{"line_number":1301,"context_line":"                        self.big_candidates[-1] \u003d self._make_stats_info("},{"line_number":1302,"context_line":"                            broker, node, own_shard_range)"},{"line_number":1303,"context_line":"                        self.big_candidates[-1].update("}],"source_content_type":"text/x-python","patch_set":4,"id":"02420ce2_a378084e","line":1300,"range":{"start_line":1300,"start_character":23,"end_line":1300,"end_character":76},"updated":"2023-02-17 15:36:29.000000000","message":"why compare active with all?\n\nif current big has [CLEAVED, CLEAVED, CREATED] that will be replaced by [ACTIVE, ACTIVE], which seems wrong","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":true,"context_lines":[{"line_number":1297,"context_line":"            # Collect the biggest root container on a cycle"},{"line_number":1298,"context_line":"            if state \u003d\u003d ShardRange.ACTIVE:"},{"line_number":1299,"context_line":"                if len(self.big_candidates) \u003e\u003d self.recon_candidates_limit:"},{"line_number":1300,"context_line":"                    if self.big_candidates[-1][\u0027active\u0027] \u003c len(shard_ranges):"},{"line_number":1301,"context_line":"                        self.big_candidates[-1] \u003d self._make_stats_info("},{"line_number":1302,"context_line":"                            broker, node, own_shard_range)"},{"line_number":1303,"context_line":"                        self.big_candidates[-1].update("}],"source_content_type":"text/x-python","patch_set":4,"id":"d7aa64db_51df92ec","line":1300,"range":{"start_line":1300,"start_character":23,"end_line":1300,"end_character":76},"in_reply_to":"02420ce2_a378084e","updated":"2023-03-10 02:22:40.000000000","message":"I\u0027ve make an assumption that the biggest root is the one with the most ACTIVE shards rather then the one with the most shards. As active means it\u0027s currently in use. One could argue so is CLEAVED but that\u0027s a temperary state, also the way this overlap code is written we count and check each state, so I\u0027d either have to completely rewrite the overlap checking (which maybe makes sense but out of scope) or pull shard ranges out of the broker yet again for an addition check (maybe that\u0027s worth it)","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"df28238091bbf3bc08ea6db96e796a44eb01c1be","unresolved":true,"context_lines":[{"line_number":1303,"context_line":"                        self.big_candidates[-1].update("},{"line_number":1304,"context_line":"                            self._get_state_count(broker))"},{"line_number":1305,"context_line":"                        self.big_candidates.sort(key\u003ditemgetter(\"active\"),"},{"line_number":1306,"context_line":"                                                 reverse\u003dTrue)"},{"line_number":1307,"context_line":"                else:"},{"line_number":1308,"context_line":"                    self.big_candidates.append(self._make_stats_info("},{"line_number":1309,"context_line":"                        broker, node, own_shard_range))"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f3ca929_fef02f8b","line":1306,"updated":"2023-02-17 15:36:29.000000000","message":"_transform_candidate_stats does the sorting and truncation to self.recon_candidates_limit, so why do it here?","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"36f85dd3866822cfafa01402c563736130474e96","unresolved":true,"context_lines":[{"line_number":1303,"context_line":"                        self.big_candidates[-1].update("},{"line_number":1304,"context_line":"                            self._get_state_count(broker))"},{"line_number":1305,"context_line":"                        self.big_candidates.sort(key\u003ditemgetter(\"active\"),"},{"line_number":1306,"context_line":"                                                 reverse\u003dTrue)"},{"line_number":1307,"context_line":"                else:"},{"line_number":1308,"context_line":"                    self.big_candidates.append(self._make_stats_info("},{"line_number":1309,"context_line":"                        broker, node, own_shard_range))"}],"source_content_type":"text/x-python","patch_set":4,"id":"3d1cac3a_5d829d29","line":1306,"in_reply_to":"5f3ca929_fef02f8b","updated":"2023-03-10 02:22:40.000000000","message":"oh it\u0027s the way I\u0027m dropping the smallest. The list needs to be sorted. So I can compare with just the smallest big candidate (last in the list), if it\u0027s bigger it belongs in the top biggest, so I can replace the lowest with the new one and sort to make sure the new smallest is on the bottom.","commit_id":"aa13026e38d1526c1ea576b6f058ed43f54d427a"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[{"line_number":524,"context_line":"                break"},{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("}],"source_content_type":"text/x-python","patch_set":2,"id":"3b906200_d1e71092","line":527,"range":{"start_line":527,"start_character":8,"end_line":527,"end_character":29},"updated":"2023-01-06 11:05:49.000000000","message":"I\u0027m a little skeptical about the usefulness of this metric because it depends on the client\u0027s request pattern and parameters  i.e. the absence of high request count metrics does not tell us that we have well-balanced shard populations, it just tells us that the clients are not making listing requests that hit any poorly balanced shards.\n\nIf the metric changes, is it because the shard population balance improved, or because the clients stopped making requests that hit poorly balanced shards?\n\nThat said, I can see it could be useful as an \u0027alarm\u0027 if metrics with high counts do appear it is a warning that shard populations may be unbalanced. But it doesn\u0027t tell us which container. Would a warning log message be better, with a configurable threshold?","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e0a4084a936d96953fe45c39c8bd58b5e5ce27a","unresolved":false,"context_lines":[{"line_number":524,"context_line":"                break"},{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("}],"source_content_type":"text/x-python","patch_set":2,"id":"18067c93_f58ab8b4","line":527,"range":{"start_line":527,"start_character":8,"end_line":527,"end_character":29},"in_reply_to":"3b906200_d1e71092","updated":"2023-01-30 02:47:00.000000000","message":"Ack","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":531,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":2,"id":"e9c9d14c_7a787e2c","line":528,"range":{"start_line":528,"start_character":13,"end_line":528,"end_character":39},"updated":"2023-01-06 11:05:49.000000000","message":"IIUC correctly the metric name will be prefixed with \u0027container\u0027 which is good.\n\nI stumble over \u0027shard_list\u0027 because we\u0027re doing an *object* listing, and (sorry to be picky!) \u0027visited\u0027 is ambiguous - the loop visits potentially more shards than requests are sent to. How about calling it exactly what it is i.e. \u0027listing.shard_requests.%d\u0027, so with the prefix it would be:\n\n  container.listing.shard_requests.[1, 2, 3,...]","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"92b6400794ecf777e5146ace85f80475c975e1a0","unresolved":true,"context_lines":[{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":531,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":2,"id":"ab68adb8_941394b3","line":528,"updated":"2023-01-06 11:05:49.000000000","message":"This isn\u0027t going to cover the cases when  a backend request is made but then an exception is raised (lines 495 and 505).\n\nEven if a refactoring results in covering those cases, the stat would be distorted because the early termination of the loop hides the fact that more backend shard range requests *would have been made* ... and if we make the leap of assuming that listings involving multiple shards (what the metric is trying to reveal) are more likely to get a bad backend response, then the metric starts to feel unreliable.\n\nI can\u0027t see an easy way around that though. We could pre-filter the shard range list to trim it to only those that match the request param constraints (marker/end_marker/prefix) but we don\u0027t know a priori how many of the remaining will be used because we\u0027ll only actually make requests until the object count limit is reached.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e0a4084a936d96953fe45c39c8bd58b5e5ce27a","unresolved":false,"context_lines":[{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":531,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":2,"id":"31b715ea_cf28701b","line":528,"in_reply_to":"ab68adb8_941394b3","updated":"2023-01-30 02:47:00.000000000","message":"yeah thy recursion that happens deeping into the call tree is cool but makes this metric (as it stands) a little inaccurate.\n\nI like the ides of tracking how many requests were made for shards for a single get reqeust, but this obviously isn\u0027t super useful. I might shelve this metric for now. We can always so some anylytics or basically get the same kind of data from the collapsible count.","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e0a4084a936d96953fe45c39c8bd58b5e5ce27a","unresolved":false,"context_lines":[{"line_number":525,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":526,"context_line":"                break"},{"line_number":527,"context_line":"        self.logger.increment("},{"line_number":528,"context_line":"            \"shard_list.visited_shards.%d\" % (len(all_resp_status)))"},{"line_number":529,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":530,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":531,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":2,"id":"2e253e89_17b5f227","line":528,"range":{"start_line":528,"start_character":13,"end_line":528,"end_character":39},"in_reply_to":"e9c9d14c_7a787e2c","updated":"2023-01-30 02:47:00.000000000","message":"Ack","commit_id":"b05be719ccba40f034dd241b5e0536677616f560"}]}