)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fca6d5d957bf758c08395f1d33f346cda04e0fdd","unresolved":true,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Change-Id: Ie15c9f40a132374c89337f0009fb5cf5a8e62c51"},{"line_number":18,"context_line":"Closes-Bug: #1928459"},{"line_number":19,"context_line":"Related-Bug: #1913332"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"a6dd8c48_2ee6a0cd","line":19,"range":{"start_line":19,"start_character":0,"end_line":19,"end_character":7},"updated":"2021-05-14 20:45:12.000000000","message":"Not \"Closes\"?","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c4074a904595d2b7f970b8d6617d6235618edee","unresolved":true,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Change-Id: Ie15c9f40a132374c89337f0009fb5cf5a8e62c51"},{"line_number":18,"context_line":"Closes-Bug: #1928459"},{"line_number":19,"context_line":"Related-Bug: #1913332"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"da9e3459_925e356c","line":19,"range":{"start_line":19,"start_character":0,"end_line":19,"end_character":7},"in_reply_to":"a6dd8c48_2ee6a0cd","updated":"2021-05-17 08:29:05.000000000","message":"I felt there was still more to the other bug e.g. see discussion here https://review.opendev.org/c/openstack/swift/+/772621/1/test/unit/container/test_sharder.py#5622\n\nbut I probably should have at least used Partial-Bug because this does remove some of the over-reporting as well as fix under-reporting.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fe0d491fce2e313b2cca9db60a8353f515ed6cc9","unresolved":true,"context_lines":[{"line_number":110,"context_line":"    result \u003d set()"},{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"}],"source_content_type":"text/x-python","patch_set":1,"id":"e015c444_f863a694","line":113,"range":{"start_line":113,"start_character":12,"end_line":113,"end_character":45},"updated":"2021-05-14 13:50:46.000000000","message":"oh, this makes it a little faster and avoid duplicate entries in the report\n\nOMM with several thousand shards this method takes O(10) seconds","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dcf7396ce2eb976adae88f7082d11f273331bfc7","unresolved":true,"context_lines":[{"line_number":110,"context_line":"    result \u003d set()"},{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"}],"source_content_type":"text/x-python","patch_set":1,"id":"a92f6c4e_2b3ed482","line":113,"range":{"start_line":113,"start_character":12,"end_line":113,"end_character":45},"in_reply_to":"29e8277e_ca5a7d38","updated":"2021-05-17 17:35:43.000000000","message":"OMM, I had a DB that needed\n\n* 10.2s before this change,\n* 6.0s with this change, and\n* 0.36s with my follow-up.\n\nNote that I only expect that sort of improvement in the common case; there are definitely cases where it\u0027ll still take a while. In particular, if I add an extra range that covers the whole namespace, I saw no real difference between this change and my follow-up. I *did* see some difference in the number of overlaps found with that extra cover-everything range, though -- I don\u0027t remember which was which now, but I saw one run report 3130 overlap sets, another with 3132, and a third with 3133.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5bbddf53ac36a7aaab5ad7b18bd00ae132d04539","unresolved":true,"context_lines":[{"line_number":110,"context_line":"    result \u003d set()"},{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"}],"source_content_type":"text/x-python","patch_set":1,"id":"29e8277e_ca5a7d38","line":113,"range":{"start_line":113,"start_character":12,"end_line":113,"end_character":45},"in_reply_to":"e015c444_f863a694","updated":"2021-05-17 07:01:15.000000000","message":"Tim I see you have a follow up to make it faster.. how much faster if you run it on the same one Al ran it on?","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9a511dcb9ad571060e2e361d87dc29139a906f6f","unresolved":true,"context_lines":[{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"},{"line_number":117,"context_line":"            overlapping.sort(key\u003dShardRange.sort_key)"}],"source_content_type":"text/x-python","patch_set":1,"id":"6506e928_be11e878","line":114,"updated":"2021-05-14 13:54:09.000000000","message":"this shard_range !\u003d sr change here seems to be the crux of the fix","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5bbddf53ac36a7aaab5ad7b18bd00ae132d04539","unresolved":true,"context_lines":[{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"},{"line_number":117,"context_line":"            overlapping.sort(key\u003dShardRange.sort_key)"}],"source_content_type":"text/x-python","patch_set":1,"id":"cc02d06b_38c1f606","line":114,"in_reply_to":"518f3d45_79d452a5","updated":"2021-05-17 07:01:15.000000000","message":"oh yeah that\u0027s right. Same upper and lower equaleth the ShardRange.. not great in this situation. Adding the name check means they\u0027ll actually cause it to check for olverlaps.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fca6d5d957bf758c08395f1d33f346cda04e0fdd","unresolved":true,"context_lines":[{"line_number":111,"context_line":"    for i, shard_range in enumerate(shard_ranges):"},{"line_number":112,"context_line":"        overlapping \u003d ["},{"line_number":113,"context_line":"            sr for sr in shard_ranges[i + 1:]"},{"line_number":114,"context_line":"            if shard_range.name !\u003d sr.name and shard_range.overlaps(sr)]"},{"line_number":115,"context_line":"        if overlapping:"},{"line_number":116,"context_line":"            overlapping.append(shard_range)"},{"line_number":117,"context_line":"            overlapping.sort(key\u003dShardRange.sort_key)"}],"source_content_type":"text/x-python","patch_set":1,"id":"518f3d45_79d452a5","line":114,"in_reply_to":"6506e928_be11e878","updated":"2021-05-14 20:45:12.000000000","message":"I wonder if something like\n\n diff --git a/swift/common/utils.py b/swift/common/utils.py\n index 950ef41a6..ba2373d1e 100644\n --- a/swift/common/utils.py\n +++ b/swift/common/utils.py\n @@ -5518,7 +5518,8 @@ class ShardRange(object):\n          # test for equality of range bounds only\n          if not isinstance(other, ShardRange):\n              return False\n -        return self.lower \u003d\u003d other.lower and self.upper \u003d\u003d other.upper\n +        return (self.lower \u003d\u003d other.lower and self.upper \u003d\u003d other.upper and\n +                self.name \u003d\u003d other.name)\n  \n      # A by-the-book implementation should probably hash the value, which\n      # in our case would be account+container+lower+upper (+timestamp ?).\n\nwould be good, though -- I don\u0027t remember why we wrote that comment. OTOH, I\u0027m also not sure where else we might test shard ranges for equality...","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d93563a9b37fe738517ee7012dcfcc589fc06a99","unresolved":true,"context_lines":[{"line_number":994,"context_line":"                # allow multiple shards in that state"},{"line_number":995,"context_line":"                continue"},{"line_number":996,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":997,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":998,"context_line":"            if overlaps:"},{"line_number":999,"context_line":"                all_overlaps \u003d \u0027, \u0027.join("},{"line_number":1000,"context_line":"                    [\u0027 \u0027.join([\u0027%s-%s\u0027 % (sr.lower, sr.upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"2d330212_e52f2642","line":997,"range":{"start_line":997,"start_character":23,"end_line":997,"end_character":60},"updated":"2021-05-14 13:48:35.000000000","message":"ideally I would retire this method and use find_paths() to discover overlaps, so that the result is same as the s-m-s-r repair command, but I was worried that would lead to all sorts of churn, and that the log messages would be hard to maintain as-is....but I also wonder how usefult the log messages are right now??","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c4074a904595d2b7f970b8d6617d6235618edee","unresolved":true,"context_lines":[{"line_number":994,"context_line":"                # allow multiple shards in that state"},{"line_number":995,"context_line":"                continue"},{"line_number":996,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":997,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":998,"context_line":"            if overlaps:"},{"line_number":999,"context_line":"                all_overlaps \u003d \u0027, \u0027.join("},{"line_number":1000,"context_line":"                    [\u0027 \u0027.join([\u0027%s-%s\u0027 % (sr.lower, sr.upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"04a0ef6c_242db51f","line":997,"range":{"start_line":997,"start_character":23,"end_line":997,"end_character":60},"in_reply_to":"04628045_fa34447a","updated":"2021-05-17 08:29:05.000000000","message":"We could used find_paths to detect overlaps and just log \"overlaps found\" with no further detail - are the bounds of interest in logs or would op just use s-m-s-r repair or analyze anyway","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5bbddf53ac36a7aaab5ad7b18bd00ae132d04539","unresolved":true,"context_lines":[{"line_number":994,"context_line":"                # allow multiple shards in that state"},{"line_number":995,"context_line":"                continue"},{"line_number":996,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":997,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":998,"context_line":"            if overlaps:"},{"line_number":999,"context_line":"                all_overlaps \u003d \u0027, \u0027.join("},{"line_number":1000,"context_line":"                    [\u0027 \u0027.join([\u0027%s-%s\u0027 % (sr.lower, sr.upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"04628045_fa34447a","line":997,"range":{"start_line":997,"start_character":23,"end_line":997,"end_character":60},"in_reply_to":"2d330212_e52f2642","updated":"2021-05-17 07:01:15.000000000","message":"We should look at using more analyse related checking in audit so they become more intergrated into the sharder. So +1 for this as a follow up, even if it\u0027s just a WIP so we can see what it\u0027ll look like.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9a511dcb9ad571060e2e361d87dc29139a906f6f","unresolved":true,"context_lines":[{"line_number":1002,"context_line":"                     for overlapping_ranges in sorted(list(overlaps))])"},{"line_number":1003,"context_line":"                warnings.append("},{"line_number":1004,"context_line":"                    \u0027overlapping ranges in state %r: %s\u0027 %"},{"line_number":1005,"context_line":"                    (ShardRange.STATES[state], all_overlaps))"},{"line_number":1006,"context_line":""},{"line_number":1007,"context_line":"        if warnings:"},{"line_number":1008,"context_line":"            self.logger.warning("}],"source_content_type":"text/x-python","patch_set":1,"id":"24c5b890_194156f5","line":1005,"updated":"2021-05-14 13:54:09.000000000","message":"one warning line seems like enough for me","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fca6d5d957bf758c08395f1d33f346cda04e0fdd","unresolved":true,"context_lines":[{"line_number":1002,"context_line":"                     for overlapping_ranges in sorted(list(overlaps))])"},{"line_number":1003,"context_line":"                warnings.append("},{"line_number":1004,"context_line":"                    \u0027overlapping ranges in state %r: %s\u0027 %"},{"line_number":1005,"context_line":"                    (ShardRange.STATES[state], all_overlaps))"},{"line_number":1006,"context_line":""},{"line_number":1007,"context_line":"        if warnings:"},{"line_number":1008,"context_line":"            self.logger.warning("}],"source_content_type":"text/x-python","patch_set":1,"id":"3287266d_af7cbc7a","line":1005,"in_reply_to":"24c5b890_194156f5","updated":"2021-05-14 20:45:12.000000000","message":"IDK -- in practice, there lines are already getting unwieldy; I\u0027m not sure I want to make them *even longer*.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c4074a904595d2b7f970b8d6617d6235618edee","unresolved":true,"context_lines":[{"line_number":1002,"context_line":"                     for overlapping_ranges in sorted(list(overlaps))])"},{"line_number":1003,"context_line":"                warnings.append("},{"line_number":1004,"context_line":"                    \u0027overlapping ranges in state %r: %s\u0027 %"},{"line_number":1005,"context_line":"                    (ShardRange.STATES[state], all_overlaps))"},{"line_number":1006,"context_line":""},{"line_number":1007,"context_line":"        if warnings:"},{"line_number":1008,"context_line":"            self.logger.warning("}],"source_content_type":"text/x-python","patch_set":1,"id":"ccd2a204_d3a29bbf","line":1005,"in_reply_to":"3287266d_af7cbc7a","updated":"2021-05-17 08:29:05.000000000","message":"There was only ever one log line, this patch just removed some of the repeated text (but added the missing overlaps)...\n\nbefore:\n  sharder-test WARNING: Audit failed for root /var/folders/n4/nf8krx110f39llj82fbw_xjm0000gp/T/tmp7q4bmalp/sda/containers/0/f33/4a8a08f09d37b73795649038408b5f33/4a8a08f09d37b73795649038408b5f33.db (a/c): overlapping ranges in state found: s-y k-t, overlapping ranges in state found: k-t s-y\n\nafter:\n  sharder-test WARNING: Audit failed for root /var/folders/n4/nf8krx110f39llj82fbw_xjm0000gp/T/tmpqw2_dr45/sda/containers/0/f33/4a8a08f09d37b73795649038408b5f33/4a8a08f09d37b73795649038408b5f33.db (a/c): overlapping ranges in state \u0027found\u0027: k-t s-y, y-z y-z","commit_id":"1b183f221d258819c82ee58961afc765529314ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dcf7396ce2eb976adae88f7082d11f273331bfc7","unresolved":true,"context_lines":[{"line_number":1002,"context_line":"                     for overlapping_ranges in sorted(list(overlaps))])"},{"line_number":1003,"context_line":"                warnings.append("},{"line_number":1004,"context_line":"                    \u0027overlapping ranges in state %r: %s\u0027 %"},{"line_number":1005,"context_line":"                    (ShardRange.STATES[state], all_overlaps))"},{"line_number":1006,"context_line":""},{"line_number":1007,"context_line":"        if warnings:"},{"line_number":1008,"context_line":"            self.logger.warning("}],"source_content_type":"text/x-python","patch_set":1,"id":"8ff52c0f_9abf7f81","line":1005,"in_reply_to":"ccd2a204_d3a29bbf","updated":"2021-05-17 17:35:43.000000000","message":"OK, yeah -- I\u0027m all for changing this to say something like\n\n Audit failed for root \u003cdb file\u003e (\u003cacc\u003e/\u003ccon\u003e): overlaps found.\n Run `swift-manage-shard-ranges \u003cdb file\u003e analyze` for more information.","commit_id":"1b183f221d258819c82ee58961afc765529314ac"}]}