)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"80d02d3e36f6e9e42413577cd4577e306a2f4b7d","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-07-31 14:44:18 -0500"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"add swift-manage-shard-ranges shink command"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"... and some new tests and code for finding acceptors"},{"line_number":10,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"9f560f44_8f6ed292","line":7,"updated":"2020-07-31 20:14:16.000000000","message":"\"shrink\"","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ed08c601cfa3ff578ad409eae32902dbda026c7","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-07-31 14:44:18 -0500"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"add swift-manage-shard-ranges shink command"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"... and some new tests and code for finding acceptors"},{"line_number":10,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"9f560f44_3c208b9b","line":7,"in_reply_to":"9f560f44_8f6ed292","updated":"2020-08-03 17:38:02.000000000","message":"Done","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"}],"bin/swift-container-sharder":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":true,"context_lines":[{"line_number":32,"context_line":"    parser.add_option(\u0027--no-auto-shard\u0027, action\u003d\u0027store_false\u0027,"},{"line_number":33,"context_line":"                      dest\u003d\u0027auto_shard\u0027, default\u003dNone,"},{"line_number":34,"context_line":"                      help\u003d\u0027Disable auto-sharding. Overrides the auto_shard \u0027"},{"line_number":35,"context_line":"                           \u0027value in the config file.\u0027)"},{"line_number":36,"context_line":"    conf_file, options \u003d parse_options(parser\u003dparser, once\u003dTrue)"},{"line_number":37,"context_line":"    run_daemon(ContainerSharder, conf_file, **options)"}],"source_content_type":"application/octet-stream","patch_set":14,"id":"8936590e_137949e9","line":35,"updated":"2020-12-18 16:36:41.000000000","message":"so i\u0027d kinda come around to the stringly-typed `--auto-shard\u003dyes` or whatever directly overriding the config precedence.","commit_id":"d277960161119face2eb9d617822b2645f19b2c1"}],"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d47f6073ce5c2665a8684436063baf3be5d197cd","unresolved":false,"context_lines":[{"line_number":424,"context_line":"                break"},{"line_number":425,"context_line":"        else:"},{"line_number":426,"context_line":"            print(\"Specified acceptor not found.\")"},{"line_number":427,"context_line":"            return 2"},{"line_number":428,"context_line":"    else:"},{"line_number":429,"context_line":"        merge_pairs \u003d find_shrinking_candidates(broker,"},{"line_number":430,"context_line":"                                                args.shrink_size,"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_845f1940","line":427,"updated":"2020-07-29 03:58:46.000000000","message":"Hrmm, being able to provide the acceptor could be a little dangerous. As it doesn\u0027t check to see if the passed in acceptor or donor are next to each other. Then down on lines 441 and 442 we use max and min, this could easily create an overlap, overlapping ranges in between.\n\nI guess when that happens the now overlapped ranges may get 2 own ranges and then cleave itself out? I guess the best thing to do is check :)\n\nEither case might make a good test :)","commit_id":"79714186abaa9bf023653b7e501c8d6ba6b9e2e6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":426,"context_line":"        acceptors[0].timestamp \u003d donor.state_timestamp"},{"line_number":427,"context_line":"    if acceptors[-1].upper \u003c donor.upper:"},{"line_number":428,"context_line":"        acceptors[-1].upper \u003d donor.upper"},{"line_number":429,"context_line":"        acceptors[-1].timestamp \u003d donor.state_timestamp"},{"line_number":430,"context_line":"    print(\u0027Donor %s\\n  %s\u0027 % (donor.name, donor))"},{"line_number":431,"context_line":"    print(\u0027Acceptors:\\n%s\u0027 % (\u0027\\n\u0027.join("},{"line_number":432,"context_line":"        \u0027  %s: %s\u0027 % (a.name, a) for a in acceptors)))"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_8c0380b8","line":429,"updated":"2020-07-31 19:20:45.000000000","message":"So we only ever care about the first and last acceptor -- and in some cases they\u0027ll be the same.","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":430,"context_line":"    print(\u0027Donor %s\\n  %s\u0027 % (donor.name, donor))"},{"line_number":431,"context_line":"    print(\u0027Acceptors:\\n%s\u0027 % (\u0027\\n\u0027.join("},{"line_number":432,"context_line":"        \u0027  %s: %s\u0027 % (a.name, a) for a in acceptors)))"},{"line_number":433,"context_line":"    choice \u003d input(\u0027Do you want to continue? [Y/n]\u0027)"},{"line_number":434,"context_line":"    if choice and choice.lower()[0] !\u003d \u0027y\u0027:"},{"line_number":435,"context_line":"        print(\u0027No changes made.\u0027)"},{"line_number":436,"context_line":"        return 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_d91a587b","line":433,"updated":"2020-07-31 19:20:45.000000000","message":"I think we\u0027ll want a --force option to bypass this, similar to what we\u0027ve got for replace.","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c428f305973e2788acbd804562f6134d3310ce19","unresolved":false,"context_lines":[{"line_number":430,"context_line":"    print(\u0027Donor %s\\n  %s\u0027 % (donor.name, donor))"},{"line_number":431,"context_line":"    print(\u0027Acceptors:\\n%s\u0027 % (\u0027\\n\u0027.join("},{"line_number":432,"context_line":"        \u0027  %s: %s\u0027 % (a.name, a) for a in acceptors)))"},{"line_number":433,"context_line":"    choice \u003d input(\u0027Do you want to continue? [Y/n]\u0027)"},{"line_number":434,"context_line":"    if choice and choice.lower()[0] !\u003d \u0027y\u0027:"},{"line_number":435,"context_line":"        print(\u0027No changes made.\u0027)"},{"line_number":436,"context_line":"        return 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_2f0ea631","line":433,"in_reply_to":"9f560f44_d91a587b","updated":"2020-07-31 19:58:47.000000000","message":"oh yeah, definitely  #willfix","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":422,"context_line":"    donor.epoch \u003d Timestamp.now()"},{"line_number":423,"context_line":"    donor.update_state(ShardRange.SHRINKING, state_timestamp\u003ddonor.epoch)"},{"line_number":424,"context_line":"    if acceptors[0].lower \u003e donor.lower:"},{"line_number":425,"context_line":"        print(\u0027*EXTENDING* first acceptor lower to cover donor\u0027)"},{"line_number":426,"context_line":"        print(\u0027  %s \u003d\u003e %s\u0027 % (acceptors[0].lower, donor.lower))"},{"line_number":427,"context_line":"        acceptors[0].lower \u003d donor.lower"},{"line_number":428,"context_line":"        acceptors[0].timestamp \u003d donor.state_timestamp"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_a27042de","line":425,"updated":"2020-10-26 18:55:59.000000000","message":"nit: s/lower/lower bound/","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":427,"context_line":"        acceptors[0].lower \u003d donor.lower"},{"line_number":428,"context_line":"        acceptors[0].timestamp \u003d donor.state_timestamp"},{"line_number":429,"context_line":"    if acceptors[-1].upper \u003c donor.upper:"},{"line_number":430,"context_line":"        print(\u0027*EXTENDING* last acceptor upper to cover donor\u0027)"},{"line_number":431,"context_line":"        print(\u0027  %s \u003d\u003e %s\u0027 % (acceptors[-1].upper, donor.upper))"},{"line_number":432,"context_line":"        acceptors[-1].upper \u003d donor.upper"},{"line_number":433,"context_line":"        acceptors[-1].timestamp \u003d donor.state_timestamp"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_42816e8f","line":430,"updated":"2020-10-26 18:55:59.000000000","message":"nit: s/upper/upper bound/","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":429,"context_line":"    if acceptors[-1].upper \u003c donor.upper:"},{"line_number":430,"context_line":"        print(\u0027*EXTENDING* last acceptor upper to cover donor\u0027)"},{"line_number":431,"context_line":"        print(\u0027  %s \u003d\u003e %s\u0027 % (acceptors[-1].upper, donor.upper))"},{"line_number":432,"context_line":"        acceptors[-1].upper \u003d donor.upper"},{"line_number":433,"context_line":"        acceptors[-1].timestamp \u003d donor.state_timestamp"},{"line_number":434,"context_line":"    print(\u0027Donor:\\n  %s: %s\u0027 % (donor.name, donor.object_count))"},{"line_number":435,"context_line":"    print(\u0027Acceptors:\\n%s\u0027 % (\u0027\\n\u0027.join("}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_c26d7e05","line":432,"updated":"2020-10-26 18:55:59.000000000","message":"this makes me nervous in light of comment here https://review.opendev.org/#/c/741721/8/swift/container/sharder.py@201\n\nis it foolproof to change a range\u0027s *upper* bound w.r.t. the naming of ranges?","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b24f3be8c73d1c1733d3b82e5f0c1f851eb7f2d","unresolved":false,"context_lines":[{"line_number":429,"context_line":"    if acceptors[-1].upper \u003c donor.upper:"},{"line_number":430,"context_line":"        print(\u0027*EXTENDING* last acceptor upper to cover donor\u0027)"},{"line_number":431,"context_line":"        print(\u0027  %s \u003d\u003e %s\u0027 % (acceptors[-1].upper, donor.upper))"},{"line_number":432,"context_line":"        acceptors[-1].upper \u003d donor.upper"},{"line_number":433,"context_line":"        acceptors[-1].timestamp \u003d donor.state_timestamp"},{"line_number":434,"context_line":"    print(\u0027Donor:\\n  %s: %s\u0027 % (donor.name, donor.object_count))"},{"line_number":435,"context_line":"    print(\u0027Acceptors:\\n%s\u0027 % (\u0027\\n\u0027.join("}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_e5c0847f","line":432,"in_reply_to":"3f65232a_c26d7e05","updated":"2020-11-03 16:56:37.000000000","message":"I think that comment must be out of date; shard range names seem to be derived purely from their account/container [0], and the container name is a function of root container, parent container, timestamp, and index [1].\n\n[0] https://github.com/openstack/swift/blob/2.26.0/swift/common/utils.py#L5029-L5031\n[1] https://github.com/openstack/swift/blob/2.26.0/swift/common/utils.py#L4988-L4996","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":439,"context_line":"        print(\u0027No changes made.\u0027)"},{"line_number":440,"context_line":"        return 1"},{"line_number":441,"context_line":"    acceptors.append(donor)"},{"line_number":442,"context_line":"    broker.merge_shard_ranges(acceptors)"},{"line_number":443,"context_line":"    print(\"Success!\")"},{"line_number":444,"context_line":""},{"line_number":445,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_6502b40b","line":442,"updated":"2020-10-26 18:55:59.000000000","message":"I wonder if we could re-use some of the code in ContainerSharder._find_and_enable_shrinking_candidates to (a) do merge into broker and (b) also push out the changes so that shrinking might kick off sooner, rather than waiting for audit","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b24f3be8c73d1c1733d3b82e5f0c1f851eb7f2d","unresolved":false,"context_lines":[{"line_number":439,"context_line":"        print(\u0027No changes made.\u0027)"},{"line_number":440,"context_line":"        return 1"},{"line_number":441,"context_line":"    acceptors.append(donor)"},{"line_number":442,"context_line":"    broker.merge_shard_ranges(acceptors)"},{"line_number":443,"context_line":"    print(\"Success!\")"},{"line_number":444,"context_line":""},{"line_number":445,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_e592449f","line":442,"in_reply_to":"3f65232a_6502b40b","updated":"2020-11-03 16:56:37.000000000","message":"I\u0027m a little torn about that idea -- I rather like that this is currently an entirely off-line process. I want to be able to\n\n* scp a busted DB to my local machine,\n* run some manage commands on it to fix it,\n* inspect it to make sure I\u0027ve got everything the way I want it,\n* then scp if back to an arbitrary handoff and let replication get it everywhere.\n\nIf/when I get to step 3 and things *aren\u0027t* how I expected, the roll-back plan is simple: delete my local copy. I can always grab a fresh copy and try again, which seems quite valuable.","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"}],"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":4911,"context_line":"    SHRINKING \u003d 50"},{"line_number":4912,"context_line":"    SHARDING \u003d 60"},{"line_number":4913,"context_line":"    SHARDED \u003d 70"},{"line_number":4914,"context_line":"    SHRUNK \u003d 80"},{"line_number":4915,"context_line":"    STATES \u003d {FOUND: \u0027found\u0027,"},{"line_number":4916,"context_line":"              CREATED: \u0027created\u0027,"},{"line_number":4917,"context_line":"              CLEAVED: \u0027cleaved\u0027,"}],"source_content_type":"text/x-python","patch_set":13,"id":"8ba313db_14002e38","line":4914,"updated":"2020-12-17 17:44:01.000000000","message":"I had to update the cheat sheet I keep on the wall!\n\nhttps://docs.google.com/document/d/1siG_vW-nAgjFL1lzo_eHUJkCAUyYYbgclK78rcb8H1s/edit?usp\u003dsharing","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":4911,"context_line":"    SHRINKING \u003d 50"},{"line_number":4912,"context_line":"    SHARDING \u003d 60"},{"line_number":4913,"context_line":"    SHARDED \u003d 70"},{"line_number":4914,"context_line":"    SHRUNK \u003d 80"},{"line_number":4915,"context_line":"    STATES \u003d {FOUND: \u0027found\u0027,"},{"line_number":4916,"context_line":"              CREATED: \u0027created\u0027,"},{"line_number":4917,"context_line":"              CLEAVED: \u0027cleaved\u0027,"}],"source_content_type":"text/x-python","patch_set":13,"id":"a3070008_2925b1d3","line":4914,"in_reply_to":"8ba313db_14002e38","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"}],"swift/container/backend.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"04bdc80e8c12ae2898b05e088f8d88b96815ec95","unresolved":false,"context_lines":[{"line_number":1778,"context_line":"        # note if this ever changes to *not* sort by upper first then it breaks"},{"line_number":1779,"context_line":"        # a key assumption for bisect, which is used by utils.find_shard_ranges"},{"line_number":1780,"context_line":"        shard_ranges.sort(key\u003dlambda sr: ("},{"line_number":1781,"context_line":"            sr.upper, sr.lower, sr.state, sr.name))"},{"line_number":1782,"context_line":"        if includes:"},{"line_number":1783,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1784,"context_line":"            return [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_a8b8b1e4","line":1781,"updated":"2020-11-11 14:39:23.000000000","message":"I think the test change about where to insert own shard range is because of this - which stabilizes on bounds over state","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":1778,"context_line":"        # note if this ever changes to *not* sort by upper first then it breaks"},{"line_number":1779,"context_line":"        # a key assumption for bisect, which is used by utils.find_shard_ranges"},{"line_number":1780,"context_line":"        shard_ranges.sort(key\u003dlambda sr: ("},{"line_number":1781,"context_line":"            sr.upper, sr.lower, sr.state, sr.name))"},{"line_number":1782,"context_line":"        if includes:"},{"line_number":1783,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1784,"context_line":"            return [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_4078918e","line":1781,"in_reply_to":"1f621f24_a8b8b1e4","updated":"2020-11-17 14:50:57.000000000","message":"what is the motivation for this change? I can\u0027t work out how it relates to the other changes in this patch.\n\nThe state was deliberately inserted ahead of lower in this change to feature/deep https://review.opendev.org/#/c/563685\nand I think it is significant in terms of where updates get directed - see comment on the corresponding test change","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d47f6073ce5c2665a8684436063baf3be5d197cd","unresolved":false,"context_lines":[{"line_number":1379,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1380,"context_line":"            else:"},{"line_number":1381,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1382,"context_line":"                break"},{"line_number":1383,"context_line":""},{"line_number":1384,"context_line":"        if not ranges_done:"},{"line_number":1385,"context_line":"            # _cleave_shard_range always store()s the context on success; make"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_e43535f9","line":1382,"updated":"2020-07-29 03:58:46.000000000","message":"nice clean up :)","commit_id":"79714186abaa9bf023653b7e501c8d6ba6b9e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8002ba481c3ba40a884c8677e75d69c7fafa885b","unresolved":false,"context_lines":[{"line_number":144,"context_line":"    return candidates"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"def find_shinrking_acceptors(donor, shard_ranges):"},{"line_number":148,"context_line":"    acceptors \u003d []"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_19acd09e","line":147,"updated":"2020-07-31 18:21:52.000000000","message":"needs a doc string","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":144,"context_line":"    return candidates"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"def find_shinrking_acceptors(donor, shard_ranges):"},{"line_number":148,"context_line":"    acceptors \u003d []"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_f99c7c5e","line":147,"in_reply_to":"9f560f44_19acd09e","updated":"2020-07-31 19:20:45.000000000","message":"And fix a typo ;-)\n\ns/find_shinrking_acceptors/find_shrinking_acceptors/","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ed08c601cfa3ff578ad409eae32902dbda026c7","unresolved":false,"context_lines":[{"line_number":144,"context_line":"    return candidates"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"def find_shinrking_acceptors(donor, shard_ranges):"},{"line_number":148,"context_line":"    acceptors \u003d []"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_fc9c53c1","line":147,"in_reply_to":"9f560f44_f99c7c5e","updated":"2020-08-03 17:38:02.000000000","message":"Done","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8002ba481c3ba40a884c8677e75d69c7fafa885b","unresolved":false,"context_lines":[{"line_number":147,"context_line":"def find_shinrking_acceptors(donor, shard_ranges):"},{"line_number":148,"context_line":"    acceptors \u003d []"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    first_lower \u003d None"},{"line_number":153,"context_line":"    for i, sr in enumerate(all_but_donor):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_99caa06a","line":150,"updated":"2020-07-31 18:21:52.000000000","message":"it depends a lot on where the list of shard_ranges comes from, but we might should filter deleted or unactive ranges","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":147,"context_line":"def find_shinrking_acceptors(donor, shard_ranges):"},{"line_number":148,"context_line":"    acceptors \u003d []"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    first_lower \u003d None"},{"line_number":153,"context_line":"    for i, sr in enumerate(all_but_donor):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_f9d47cbf","line":150,"in_reply_to":"9f560f44_99caa06a","updated":"2020-07-31 19:20:45.000000000","message":"Should we support shrinking everything back into the root? If we want to, note that this list may be empty... so first_lower could be None.","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":150,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name]"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    first_lower \u003d None"},{"line_number":153,"context_line":"    for i, sr in enumerate(all_but_donor):"},{"line_number":154,"context_line":"        if sr.lower \u003c\u003d donor.lower:"},{"line_number":155,"context_line":"            if not first_lower:"},{"line_number":156,"context_line":"                first_lower \u003d sr"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_b9e044ce","line":153,"range":{"start_line":153,"start_character":27,"end_line":153,"end_character":40},"updated":"2020-07-31 19:20:45.000000000","message":"What assumptions are we making with regard to order?","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":159,"context_line":"            # else sr.lower \u003d\u003d first_lower.lower i.e. *second* lower :P"},{"line_number":160,"context_line":"            continue"},{"line_number":161,"context_line":"        if not first_lower:"},{"line_number":162,"context_line":"            assert donor.lower \u003d\u003d ShardRange.MIN"},{"line_number":163,"context_line":"            first_lower \u003d sr"},{"line_number":164,"context_line":"        break"},{"line_number":165,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_1984d0d3","line":162,"updated":"2020-07-31 19:20:45.000000000","message":"So if you\u0027ve got two non-overlapping shard ranges like\n\n \u0027\u0027 - \u0027foobar\u0027, 10 rows\n \u0027foobar\u0027 - \u0027\u0027, 100K rows\n\nYou want to shrink that first one since it\u0027s so tiny... but won\u0027t this assertion fail?","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c428f305973e2788acbd804562f6134d3310ce19","unresolved":false,"context_lines":[{"line_number":159,"context_line":"            # else sr.lower \u003d\u003d first_lower.lower i.e. *second* lower :P"},{"line_number":160,"context_line":"            continue"},{"line_number":161,"context_line":"        if not first_lower:"},{"line_number":162,"context_line":"            assert donor.lower \u003d\u003d ShardRange.MIN"},{"line_number":163,"context_line":"            first_lower \u003d sr"},{"line_number":164,"context_line":"        break"},{"line_number":165,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_cc1a987b","line":162,"in_reply_to":"9f560f44_1984d0d3","updated":"2020-07-31 19:58:47.000000000","message":"I didn\u0027t actually mean to leave this assertion in here, and this was originally: raise Exception(\"I\u0027m too dumb\")\n\nI think you\u0027re right, this edge case could use more tests - but the special case of shrinking to the right IS specifically for when donor.lower is MIN","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4017bcf699bca4c2122faf91c8f4e316d4385ac1","unresolved":false,"context_lines":[{"line_number":159,"context_line":"            # else sr.lower \u003d\u003d first_lower.lower i.e. *second* lower :P"},{"line_number":160,"context_line":"            continue"},{"line_number":161,"context_line":"        if not first_lower:"},{"line_number":162,"context_line":"            assert donor.lower \u003d\u003d ShardRange.MIN"},{"line_number":163,"context_line":"            first_lower \u003d sr"},{"line_number":164,"context_line":"        break"},{"line_number":165,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_afc116e7","line":162,"in_reply_to":"9f560f44_cc1a987b","updated":"2020-07-31 20:49:47.000000000","message":"Oh! My bad -- read this as checking sr.lower \u003d\u003d MIN. You\u0027re right, this is already covered in tests.","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":163,"context_line":"            first_lower \u003d sr"},{"line_number":164,"context_line":"        break"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    acceptors.append(first_lower)"},{"line_number":167,"context_line":"    if donor.upper \u003c\u003d first_lower.upper:"},{"line_number":168,"context_line":"        # fully overlapping first_lower range"},{"line_number":169,"context_line":"        return acceptors"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_59b2c895","line":166,"range":{"start_line":166,"start_character":21,"end_line":166,"end_character":32},"updated":"2020-07-31 19:20:45.000000000","message":"OK, that loop was a bit much to digest, let me see if I\u0027ve got this right:\n\nfirst_lower should be either\n\n* the last (non-donor) shard range whose lower is \u003c\u003d donor\u0027s lower, or\n* the first (non-donor) shard range if there is no shard range that satisfies ^^^","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c428f305973e2788acbd804562f6134d3310ce19","unresolved":false,"context_lines":[{"line_number":163,"context_line":"            first_lower \u003d sr"},{"line_number":164,"context_line":"        break"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    acceptors.append(first_lower)"},{"line_number":167,"context_line":"    if donor.upper \u003c\u003d first_lower.upper:"},{"line_number":168,"context_line":"        # fully overlapping first_lower range"},{"line_number":169,"context_line":"        return acceptors"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_0c15b04a","line":166,"range":{"start_line":166,"start_character":21,"end_line":166,"end_character":32},"in_reply_to":"9f560f44_59b2c895","updated":"2020-07-31 19:58:47.000000000","message":"sounds right to me!","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"883ade5974cf60800159caba540a21747f76516c","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    for sr in all_but_donor[i - 1:]:"},{"line_number":174,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":175,"context_line":"            break"},{"line_number":176,"context_line":"        if sr.lower \u003d\u003d end:"},{"line_number":177,"context_line":"            end \u003d sr.upper"},{"line_number":178,"context_line":"            acceptors.append(sr)"},{"line_number":179,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_0c5bf0f6","line":176,"updated":"2020-07-31 19:20:45.000000000","message":"What if there are multiple shards the match on this? I guess, we just pick one and keep going... Yeah, that\u0027s probably fine.\n\nIt\u0027s a good thing there should never be any dead-ends!","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c428f305973e2788acbd804562f6134d3310ce19","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    for sr in all_but_donor[i - 1:]:"},{"line_number":174,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":175,"context_line":"            break"},{"line_number":176,"context_line":"        if sr.lower \u003d\u003d end:"},{"line_number":177,"context_line":"            end \u003d sr.upper"},{"line_number":178,"context_line":"            acceptors.append(sr)"},{"line_number":179,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_0f0ac21f","line":176,"in_reply_to":"9f560f44_0c5bf0f6","updated":"2020-07-31 19:58:47.000000000","message":"I mean... we don\u0027t *expect* dead ends and this code is broken if there is... so yeah \"should never\" FLW","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8002ba481c3ba40a884c8677e75d69c7fafa885b","unresolved":false,"context_lines":[{"line_number":177,"context_line":"            end \u003d sr.upper"},{"line_number":178,"context_line":"            acceptors.append(sr)"},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"    return acceptors"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"def find_shrinking_candidates(broker, shrink_threshold, merge_size):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_d9a1b8ab","line":180,"updated":"2020-07-31 18:21:52.000000000","message":"there\u0027s a number of sanity assertions we could make about state, and contiguousness, etc","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4017bcf699bca4c2122faf91c8f4e316d4385ac1","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    acceptors \u003d []"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name"},{"line_number":154,"context_line":"                     and sr.state \u003d\u003d ShardRange.ACTIVE and not sr.deleted]"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    first_lower \u003d None"},{"line_number":157,"context_line":"    for i, sr in enumerate(all_but_donor):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_6fc1be2e","line":154,"range":{"start_line":154,"start_character":37,"end_line":154,"end_character":54},"updated":"2020-07-31 20:49:47.000000000","message":"Might want to be willing to shrink into CLEAVED, too. Otherwise we can\u0027t resolve a split-brain where two roots independently establish shard ranges -- only when two shards do it.","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ed08c601cfa3ff578ad409eae32902dbda026c7","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    acceptors \u003d []"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name"},{"line_number":154,"context_line":"                     and sr.state \u003d\u003d ShardRange.ACTIVE and not sr.deleted]"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    first_lower \u003d None"},{"line_number":157,"context_line":"    for i, sr in enumerate(all_but_donor):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_fc77339f","line":154,"range":{"start_line":154,"start_character":37,"end_line":154,"end_character":54},"in_reply_to":"9f560f44_6fc1be2e","updated":"2020-08-03 17:38:02.000000000","message":"Done","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"737bf96de255adb795603c03203ce70c82580681","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    acceptors \u003d []"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name"},{"line_number":154,"context_line":"                     and sr.state \u003d\u003d ShardRange.ACTIVE and not sr.deleted]"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    first_lower \u003d None"},{"line_number":157,"context_line":"    for i, sr in enumerate(all_but_donor):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_52196820","line":154,"range":{"start_line":154,"start_character":37,"end_line":154,"end_character":54},"in_reply_to":"9f560f44_fc77339f","updated":"2020-08-28 19:41:04.000000000","message":"I\u0027m not sure the test I wrote to prove we shrink into a CLEAVED shard is truly representative of the scenario where you think this behavior would be useful; but I went ahead and made the change","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9b93e0eaa493975220141eaf0a1cb200bd1dc655","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"    # extend a contiguous range over donor"},{"line_number":176,"context_line":"    end \u003d first_lower.upper"},{"line_number":177,"context_line":"    for sr in all_but_donor[i - 1:]:"},{"line_number":178,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":179,"context_line":"            break"},{"line_number":180,"context_line":"        if sr.lower \u003d\u003d end:"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_2b861667","line":177,"updated":"2020-08-11 06:55:54.000000000","message":"Just trying to decide if it would at all possible for `i` to equal 0 here, because that would only ever check the last item in the all_but_donor list.\n\nI guess that can only happen if the first (SR in the top loop) SR.lower isn\u0027t lower then the doner BUT the first SR.upper \u003c doner.upper.\n\nIf that ever does happen, we could possibly get a whopping big hole.","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ed0829dadbcaa165ec236188dfd8c06e31593e06","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"    # extend a contiguous range over donor"},{"line_number":176,"context_line":"    end \u003d first_lower.upper"},{"line_number":177,"context_line":"    for sr in all_but_donor[i - 1:]:"},{"line_number":178,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":179,"context_line":"            break"},{"line_number":180,"context_line":"        if sr.lower \u003d\u003d end:"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_a671b8e1","line":177,"in_reply_to":"9f560f44_2b861667","updated":"2020-08-11 16:15:30.000000000","message":"I\u0027d *love* for this algorithm to be more clear - I feel a lot better about the *tests* than I do this code.\n\nAnyway I think you\u0027re right and I can write a test to show there\u0027s a bug here - well spotted! #willfix","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"737bf96de255adb795603c03203ce70c82580681","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"    # extend a contiguous range over donor"},{"line_number":176,"context_line":"    end \u003d first_lower.upper"},{"line_number":177,"context_line":"    for sr in all_but_donor[i - 1:]:"},{"line_number":178,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":179,"context_line":"            break"},{"line_number":180,"context_line":"        if sr.lower \u003d\u003d end:"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_32683486","line":177,"in_reply_to":"9f560f44_a671b8e1","updated":"2020-08-28 19:41:04.000000000","message":"The test case I found to trigger i \u003d\u003d 0 looked like\n\n\t\t      |-----|\n\tdonor -\u003e |----------------|\n\t\t\t    |-----|\n\n\nessentially when we pick the `first_lower \u003d sr` special case by fiat we have to advance the pointer i \u003d 1 manually so we can connect it to any other acceptors we need to cover the range.\n\nExcellent nose Matt!","commit_id":"fffbb9cffd70361e4dcd6cdd2260dec2be5b3e31"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2173b4bf985e72b9e6c12eff9a1feca2085260c6","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    acceptors \u003d []"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    all_but_donor \u003d [sr for sr in shard_ranges if sr.name !\u003d donor.name"},{"line_number":154,"context_line":"                     and sr.state in (ShardRange.ACTIVE, ShardRange.CLEAVED)"},{"line_number":155,"context_line":"                     and not sr.deleted]"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    first_lower \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"9f560f44_3f7d6fb2","line":154,"updated":"2020-10-02 00:37:39.000000000","message":"*grumble grumble*\n\nOK -- ACTIVE, CLEAVED, or *CREATED*. My probe test was hitting trouble (in part) because it\u0027d have some overlaps like\n\n CLEAVED   CLEAVED   CREATED\n CLEAVED   CLEAVED   CREATED\n\nAnd the sharder would stay wedged even once I got those to\n\n CLEAVED   CLEAVED   CREATED\n SHRINKING SHRINKING CREATED","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    first_lower \u003d None"},{"line_number":158,"context_line":"    index \u003d 1"},{"line_number":159,"context_line":"    for i, sr in enumerate(all_but_donor):"},{"line_number":160,"context_line":"        if sr.lower \u003c\u003d donor.lower:"},{"line_number":161,"context_line":"            if not first_lower:"},{"line_number":162,"context_line":"                index \u003d i"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_62790a92","line":159,"updated":"2020-10-26 18:55:59.000000000","message":"this does rely on the shard_ranges being sorted, but this is a public interface so there is no guarantee they will be, so that requirement/expectation should at least be documented, or maybe shard_ranges should be sorted in this method for good measure?","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":168,"context_line":"            continue"},{"line_number":169,"context_line":"        if not first_lower:"},{"line_number":170,"context_line":"            # because of order this only happens when"},{"line_number":171,"context_line":"            #   donor.lower \u003d\u003d ShardRange.MIN"},{"line_number":172,"context_line":"            index \u003d 1"},{"line_number":173,"context_line":"            first_lower \u003d sr"},{"line_number":174,"context_line":"            break"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_ff2493c8","line":171,"updated":"2020-10-26 18:55:59.000000000","message":"not strictly true because the broker\u0027s get_shard_ranges sorts by upper bound first. Consider for example:\n\n      def test_first_shrinks_to_the_right_2(self):\n        epoch \u003d next(self.ts)\n\n        shard_ranges \u003d [\n            ShardRange(self.srn(epoch, 0), epoch, \u0027a\u0027, \u0027az\u0027,\n                       object_count\u003d100, state\u003dShardRange.ACTIVE),\n            ShardRange(self.srn(epoch, 1), epoch, \u0027ay\u0027, \u0027az\u0027,\n                       object_count\u003d100, state\u003dShardRange.ACTIVE),\n            ShardRange(self.srn(epoch, 2), epoch, ShardRange.MIN, \u0027b\u0027,\n                       object_count\u003d100, state\u003dShardRange.ACTIVE),\n            ShardRange(self.srn(epoch, 3), epoch, \u0027b\u0027, \u0027c\u0027,\n                       object_count\u003d100, state\u003dShardRange.ACTIVE),\n            ShardRange(self.srn(epoch, 4), epoch, \u0027c\u0027, ShardRange.MAX,\n                       object_count\u003d100, state\u003dShardRange.ACTIVE),\n        ]\n        shard_ranges \u003d self._stabilize(shard_ranges)\n\n        acceptors \u003d find_shrinking_acceptors(shard_ranges[0], shard_ranges)\n        self.assertEqual(acceptors, [shard_ranges[1]])","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":184,"context_line":"        if end \u003e\u003d donor.upper:"},{"line_number":185,"context_line":"            break"},{"line_number":186,"context_line":"        if sr.lower \u003e\u003d donor.upper:"},{"line_number":187,"context_line":"            continue"},{"line_number":188,"context_line":"        if sr.lower \u003d\u003d end:"},{"line_number":189,"context_line":"            end \u003d sr.upper"},{"line_number":190,"context_line":"            acceptors.append(sr)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_c25a9ef1","line":187,"updated":"2020-10-26 18:55:59.000000000","message":"given there is only one subsequent condition, the if-then-continue could be dropped\n\nBut, if there is a gap in shard ranges, should/could that be fixed first?\n\nAlso - what about `sr.lower \u003c end` ? would a shrink into overlapping ranges work? if it remains ignored, the condition might at least be worth a comment","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2173b4bf985e72b9e6c12eff9a1feca2085260c6","unresolved":false,"context_lines":[{"line_number":785,"context_line":"                include_deleted\u003dTrue)"},{"line_number":786,"context_line":"            if shard_ranges:"},{"line_number":787,"context_line":"                for shard_range in shard_ranges:"},{"line_number":788,"context_line":"                    if (shard_range.lower \u003d\u003d own_shard_range.lower and"},{"line_number":789,"context_line":"                            shard_range.upper \u003d\u003d own_shard_range.upper and"},{"line_number":790,"context_line":"                            shard_range.name \u003d\u003d own_shard_range.name):"},{"line_number":791,"context_line":"                        break"},{"line_number":792,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"9f560f44_bf105f5b","line":789,"range":{"start_line":788,"start_character":24,"end_line":789,"end_character":70},"updated":"2020-10-02 00:37:39.000000000","message":"Manual shrinking can shift bounds at the root without pushing the new bounds out immediately -- this seems way too constraining; it should probably just be a name check now.","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":813,"context_line":""},{"line_number":814,"context_line":"        if shard_range:"},{"line_number":815,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":816,"context_line":"                              len(shard_ranges))"},{"line_number":817,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":818,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":819,"context_line":"            delete_age \u003d time.time() - self.reclaim_age"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_82a1c6ac","line":816,"updated":"2020-10-26 18:55:59.000000000","message":"My IDE whines that shard_ranges may not be assigned. It\u0027s a false positive because shard_range is None unless shard_ranges has been assigned, but might be nice to refactor. The use of var name \u0027shard_range\u0027 is also a little confusing here - I think it refers to the parent\u0027s version of this shard\u0027s own range?","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":814,"context_line":"        if shard_range:"},{"line_number":815,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":816,"context_line":"                              len(shard_ranges))"},{"line_number":817,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":818,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":819,"context_line":"            delete_age \u003d time.time() - self.reclaim_age"},{"line_number":820,"context_line":"            if (own_shard_range.state \u003d\u003d ShardRange.SHARDED and"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_c299fe6e","line":817,"updated":"2020-10-26 18:55:59.000000000","message":"that feels like a significant change - so I guess shard_ranges is bounded by this shard\u0027s lower/upper so the idea is to install any shrinking acceptors that have been fetched from root? But the acceptors\u0027 ranges will be beyond this shard\u0027s bounds so how do they get included in the fetched shard_ranges given the marker/end_marker....ooh, so the fetch includes ranges that *include* marker/end_marker.\n\nI definitely think a comment would be worthwhile here, especially since this is \u0027hidden\u0027 in the *audit* function","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7b2547dbfcc472d7fb2a9080889312f45328fda4","unresolved":false,"context_lines":[{"line_number":814,"context_line":"        if shard_range:"},{"line_number":815,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":816,"context_line":"                              len(shard_ranges))"},{"line_number":817,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":818,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":819,"context_line":"            delete_age \u003d time.time() - self.reclaim_age"},{"line_number":820,"context_line":"            if (own_shard_range.state \u003d\u003d ShardRange.SHARDED and"}],"source_content_type":"text/x-python","patch_set":8,"id":"1f621f24_27d55044","line":817,"in_reply_to":"3f65232a_c299fe6e","updated":"2020-11-03 16:39:51.000000000","message":"shit yes, this is *critical* we need to break this out to a separate change so we can merge at least THIS part jeez","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2173b4bf985e72b9e6c12eff9a1feca2085260c6","unresolved":false,"context_lines":[{"line_number":1426,"context_line":"                # else, no errors, but no rows found either. keep going,"},{"line_number":1427,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1428,"context_line":"            else:"},{"line_number":1429,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1430,"context_line":"                break"},{"line_number":1431,"context_line":""},{"line_number":1432,"context_line":"        if not ranges_done:"}],"source_content_type":"text/x-python","patch_set":8,"id":"9f560f44_e5c6ca69","line":1429,"updated":"2020-10-02 00:37:39.000000000","message":"I think we should skip over anybody that\u0027s shrinking; no rows should be moving there anyway. Or maybe we should just query for shards with \u0027update\u0027 states up at L1398?","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":693,"context_line":""},{"line_number":694,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):"},{"line_number":695,"context_line":"            shard_ranges \u003d [sr for sr in broker.get_shard_ranges()"},{"line_number":696,"context_line":"                            if sr.state !\u003d ShardRange.SHRINKING]"},{"line_number":697,"context_line":"            missing_ranges \u003d find_missing_ranges(shard_ranges)"},{"line_number":698,"context_line":"            if missing_ranges:"},{"line_number":699,"context_line":"                warnings.append("}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_526ba3ef","line":696,"updated":"2020-11-10 00:30:21.000000000","message":"I think this one needs test coverage.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":702,"context_line":"                              for lower, upper in missing_ranges]))"},{"line_number":703,"context_line":""},{"line_number":704,"context_line":"        for state in ShardRange.STATES:"},{"line_number":705,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":706,"context_line":"                # Shrinking is how we resolve overlaps; we\u0027ve got to"},{"line_number":707,"context_line":"                # allow multiple shards in that state"},{"line_number":708,"context_line":"                continue"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_d29cd36b","line":705,"updated":"2020-11-10 00:30:21.000000000","message":"Should maybe also have a carve-out for SHARDING -- IIRC, we pretty quickly move from\n\n* SHRINKING to\n* SHARDING to\n* SHARDED (deleted)\n\nthough since you\u0027re unlikely to be shrinking into more than two other shards, maybe that SHARDING to SHARDED/deleted state change happens quickly enough that we don\u0027t much care.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":773,"context_line":"        if shard_range:"},{"line_number":774,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":775,"context_line":"                              len(shard_ranges))"},{"line_number":776,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":777,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":778,"context_line":""},{"line_number":779,"context_line":"        delete_age \u003d time.time() - self.reclaim_age"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_d27e93ad","line":776,"updated":"2020-11-10 00:30:21.000000000","message":"Needs test coverage.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":1374,"context_line":"        ranges_done \u003d []"},{"line_number":1375,"context_line":"        for shard_range in ranges_todo:"},{"line_number":1376,"context_line":"            if shard_range.state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":1377,"context_line":"                continue"},{"line_number":1378,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1379,"context_line":"                                     ShardRange.CLEAVED,"},{"line_number":1380,"context_line":"                                     ShardRange.ACTIVE):"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_124e4b96","line":1377,"updated":"2020-11-10 00:30:21.000000000","message":"Needs test coverage.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"010d881dc83e033ac2a3e983b033f1c178942c54","unresolved":false,"context_lines":[{"line_number":1376,"context_line":"            if shard_range.state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":1377,"context_line":"                continue"},{"line_number":1378,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1379,"context_line":"                                     ShardRange.CLEAVED,"},{"line_number":1380,"context_line":"                                     ShardRange.ACTIVE):"},{"line_number":1381,"context_line":"                cleave_result \u003d self._cleave_shard_range("},{"line_number":1382,"context_line":"                    broker, cleaving_context, shard_range)"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_4d5826f9","line":1379,"updated":"2020-11-10 01:05:04.000000000","message":"pep8: E128 continuation line under-indented for visual indent","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"010d881dc83e033ac2a3e983b033f1c178942c54","unresolved":false,"context_lines":[{"line_number":1377,"context_line":"                continue"},{"line_number":1378,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1379,"context_line":"                                     ShardRange.CLEAVED,"},{"line_number":1380,"context_line":"                                     ShardRange.ACTIVE):"},{"line_number":1381,"context_line":"                cleave_result \u003d self._cleave_shard_range("},{"line_number":1382,"context_line":"                    broker, cleaving_context, shard_range)"},{"line_number":1383,"context_line":"                if cleave_result \u003d\u003d CLEAVE_SUCCESS:"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_2d5deae8","line":1380,"updated":"2020-11-10 01:05:04.000000000","message":"pep8: E128 continuation line under-indented for visual indent","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":1389,"context_line":"                # else, no errors, but no rows found either. keep going,"},{"line_number":1390,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1391,"context_line":"            else:"},{"line_number":1392,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1393,"context_line":"                break"},{"line_number":1394,"context_line":""},{"line_number":1395,"context_line":"        if not ranges_done:"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_f252af3a","line":1392,"range":{"start_line":1392,"start_character":60,"end_line":1392,"end_character":62},"updated":"2020-11-10 00:30:21.000000000","message":"Needs test coverage.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e9370df7798fb6e6a231ef6819c96a7018abc21c","unresolved":false,"context_lines":[{"line_number":749,"context_line":"                        break"},{"line_number":750,"context_line":"                else:"},{"line_number":751,"context_line":"                    # this is not necessarily an error - some replicas of the"},{"line_number":752,"context_line":"                    # root may not yet know about this shard container"},{"line_number":753,"context_line":"                    warnings.append(\u0027root has no matching shard range\u0027)"},{"line_number":754,"context_line":"                    shard_range \u003d None"},{"line_number":755,"context_line":"            elif not own_shard_range.deleted:"}],"source_content_type":"text/x-python","patch_set":10,"id":"1f621f24_3007f584","line":752,"updated":"2020-11-10 16:49:20.000000000","message":"So previously, we\u0027d assume that we have newer information than the root and we wouldn\u0027t update... but now we\u0027ll merge in the shards that the root has (which will most likely just be this shard\u0027s parent shard, either in ACTIVE or SHARDING state)...\n\nBut regardless of whether that causes audit failures for this subshard, the parent shard will keep sharding and eventually update the root with both its new sharded state and all the subshard info. Then the next time this guy comes around, he\u0027ll get the updated states and continue normally.","commit_id":"5e0ad7119910aa655a736d2e7447b7b08f19590d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e9370df7798fb6e6a231ef6819c96a7018abc21c","unresolved":false,"context_lines":[{"line_number":770,"context_line":"            self._increment_stat(\u0027audit_shard\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":771,"context_line":"            return False"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":"        if shard_ranges and not errors:"},{"line_number":774,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":775,"context_line":"                              len(shard_ranges))"},{"line_number":776,"context_line":"            broker.merge_shard_ranges(shard_ranges)"}],"source_content_type":"text/x-python","patch_set":10,"id":"1f621f24_3061b5aa","line":773,"range":{"start_line":773,"start_character":24,"end_line":773,"end_character":38},"updated":"2020-11-10 16:49:20.000000000","message":"Meh -- if there were errors, we would\u0027ve returned already in the stanza above.\n\nThe shard_range vs shard_ranges distinction is subtle... I think I\u0027m ok with it? See notes above.","commit_id":"5e0ad7119910aa655a736d2e7447b7b08f19590d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":693,"context_line":""},{"line_number":694,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):"},{"line_number":695,"context_line":"            shard_ranges \u003d [sr for sr in broker.get_shard_ranges()"},{"line_number":696,"context_line":"                            if sr.state !\u003d ShardRange.SHRINKING]"},{"line_number":697,"context_line":"            missing_ranges \u003d find_missing_ranges(shard_ranges)"},{"line_number":698,"context_line":"            if missing_ranges:"},{"line_number":699,"context_line":"                warnings.append("}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_fba511a0","line":696,"updated":"2020-11-17 14:50:57.000000000","message":"+1, a shrinking shard range is on its way to being a missing shard range unless it overlaps with other ranges, so let\u0027s not include shrinking shards when checking coverage\n\nBUT - this change is untested","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":705,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":706,"context_line":"                # Shrinking is how we resolve overlaps; we\u0027ve got to"},{"line_number":707,"context_line":"                # allow multiple shards in that state"},{"line_number":708,"context_line":"                continue"},{"line_number":709,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":710,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":711,"context_line":"            for overlapping_ranges in overlaps:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_bb91d954","line":708,"updated":"2020-11-17 14:50:57.000000000","message":"+1, for \u0027simple\u0027 shrinking within a single continuous path of shard ranges we did not expect two shrinking shard ranges to overlap. However, for sets of shards with multiple, possibly incomplete, paths of shard ranges we may wish to simultaneously shrink multiple overlapping shards.","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":736,"context_line":""},{"line_number":737,"context_line":"        own_shard_range \u003d broker.get_own_shard_range(no_default\u003dTrue)"},{"line_number":738,"context_line":""},{"line_number":739,"context_line":"        own_shard_range_from_root \u003d None"},{"line_number":740,"context_line":"        if own_shard_range:"},{"line_number":741,"context_line":"            shard_ranges \u003d self._fetch_shard_ranges("},{"line_number":742,"context_line":"                broker, newest\u003dTrue,"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_9be85d7c","line":739,"updated":"2020-11-17 14:50:57.000000000","message":"+1","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":746,"context_line":"            if shard_ranges:"},{"line_number":747,"context_line":"                for shard_range in shard_ranges:"},{"line_number":748,"context_line":"                    if shard_range.name \u003d\u003d own_shard_range.name:"},{"line_number":749,"context_line":"                        own_shard_range_from_root \u003d shard_range"},{"line_number":750,"context_line":"                        break"},{"line_number":751,"context_line":"                else:"},{"line_number":752,"context_line":"                    # this is not necessarily an error - some replicas of the"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_5b1c0577","line":749,"updated":"2020-11-17 14:50:57.000000000","message":"I\u0027m not sure about this change. I assume the intention is to enable acceptors to learn their new expanded bounds from the root? But consider an acceptor whose shard range has already been expanded - how can we be confident that *every* root has the updated upper and lower for the acceptor? It seems there is a risk that the expanded acceptor will match on name only and take stale bounds from an out-of-date root, and compress its bounds.\n\nThe current shrinking code pushes the expanded acceptor shard range from the \u0027master\u0027 root to the shard containers...but I\u0027m not sure if there is a recovery path if that initial push fails, so something like this change does seem necessary.\n\nDoes the timestamp save us from inadvertent rollback of the acceptor bounds? The current shrink code *does* update the acceptor timestamp when expanding its bounds, so perhaps we only accept the own_shard_range_from_root if timestamp \u003e\u003d own_shard_range and all is good.","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":747,"context_line":"                for shard_range in shard_ranges:"},{"line_number":748,"context_line":"                    if shard_range.name \u003d\u003d own_shard_range.name:"},{"line_number":749,"context_line":"                        own_shard_range_from_root \u003d shard_range"},{"line_number":750,"context_line":"                        break"},{"line_number":751,"context_line":"                else:"},{"line_number":752,"context_line":"                    # this is not necessarily an error - some replicas of the"},{"line_number":753,"context_line":"                    # root may not yet know about this shard container"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_5697bcec","line":750,"updated":"2020-11-17 14:50:57.000000000","message":"we could retain the comparison of upper and lower for non-ACTIVE shards i.e. only relax the match criteria for shard ranges we expect to have changing bounds","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":773,"context_line":"        if own_shard_range_from_root:"},{"line_number":774,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":775,"context_line":"                              len(shard_ranges))"},{"line_number":776,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":777,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":778,"context_line":""},{"line_number":779,"context_line":"        delete_age \u003d time.time() - self.reclaim_age"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_f6a528a0","line":776,"updated":"2020-11-17 14:50:57.000000000","message":"I guess the intention here is to enable shrinking shards to learn about their acceptors?\n\nCan we restrict to only merging other non-shrinking ranges if own range is shrinking? otherwise we just add clutter to the shard db\u0027s: acceptors don\u0027t need to know about donors, and overlapping shrinkers don\u0027t need to know about other shrinkers - if nothing else that adds noise when debugging.","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba5b69ba418a0a7bdf0de57f196054a859ce1025","unresolved":false,"context_lines":[{"line_number":1374,"context_line":"        ranges_done \u003d []"},{"line_number":1375,"context_line":"        for shard_range in ranges_todo:"},{"line_number":1376,"context_line":"            if shard_range.state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":1377,"context_line":"                continue"},{"line_number":1378,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1379,"context_line":"                                       ShardRange.CLEAVED,"},{"line_number":1380,"context_line":"                                       ShardRange.ACTIVE):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_44d47320","line":1377,"updated":"2020-11-10 23:05:51.000000000","message":"Good call, otherwise we may stop at a shardrange that is marked as shrinking and not continue to actually do any more cleaving.","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":1374,"context_line":"        ranges_done \u003d []"},{"line_number":1375,"context_line":"        for shard_range in ranges_todo:"},{"line_number":1376,"context_line":"            if shard_range.state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":1377,"context_line":"                continue"},{"line_number":1378,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1379,"context_line":"                                       ShardRange.CLEAVED,"},{"line_number":1380,"context_line":"                                       ShardRange.ACTIVE):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_b6c8d054","line":1377,"in_reply_to":"1f621f24_44d47320","updated":"2020-11-17 14:50:57.000000000","message":"ok, but I would still advocate not merging other shrinkers into a shrinking shard","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":1389,"context_line":"                # else, no errors, but no rows found either. keep going,"},{"line_number":1390,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1391,"context_line":"            else:"},{"line_number":1392,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1393,"context_line":"                break"},{"line_number":1394,"context_line":""},{"line_number":1395,"context_line":"        if not ranges_done:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_f6bec8ab","line":1392,"updated":"2020-11-17 14:50:57.000000000","message":"We could retain the warning if state !\u003d FOUND","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba5b69ba418a0a7bdf0de57f196054a859ce1025","unresolved":false,"context_lines":[{"line_number":1390,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1391,"context_line":"            else:"},{"line_number":1392,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1393,"context_line":"                break"},{"line_number":1394,"context_line":""},{"line_number":1395,"context_line":"        if not ranges_done:"},{"line_number":1396,"context_line":"            # _cleave_shard_range always store()s the context on success; make"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_84da0b48","line":1393,"updated":"2020-11-10 23:05:51.000000000","message":"Oh and this catches the FOUND, nice :)","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":693,"context_line":""},{"line_number":694,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):"},{"line_number":695,"context_line":"            shard_ranges \u003d [sr for sr in broker.get_shard_ranges()"},{"line_number":696,"context_line":"                            if sr.state !\u003d ShardRange.SHRINKING]"},{"line_number":697,"context_line":"            missing_ranges \u003d find_missing_ranges(shard_ranges)"},{"line_number":698,"context_line":"            if missing_ranges:"},{"line_number":699,"context_line":"                warnings.append("}],"source_content_type":"text/x-python","patch_set":12,"id":"224c8d3d_b98966b1","line":696,"updated":"2020-11-24 22:46:01.000000000","message":"for sure shrinking shards don\u0027t help us make up any gaps","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":705,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":706,"context_line":"                # Shrinking is how we resolve overlaps; we\u0027ve got to"},{"line_number":707,"context_line":"                # allow multiple shards in that state"},{"line_number":708,"context_line":"                continue"},{"line_number":709,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":710,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":711,"context_line":"            for overlapping_ranges in overlaps:"}],"source_content_type":"text/x-python","patch_set":12,"id":"210e2b91_ac134a23","line":708,"updated":"2020-11-24 22:46:01.000000000","message":"I agree - shinking ranges can overlap for all I care - if you have a three way split it\u0027s obvious to see you\u0027ll have multiple shrinkers overlapping with one active\n\nno one cares if shrinking ranges overlap, they\u0027re all going away anyway!","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":707,"context_line":"                # allow multiple shards in that state"},{"line_number":708,"context_line":"                continue"},{"line_number":709,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":710,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":711,"context_line":"            for overlapping_ranges in overlaps:"},{"line_number":712,"context_line":"                warnings.append("},{"line_number":713,"context_line":"                    \u0027overlapping ranges in state %s: %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":12,"id":"8b62ff90_20294e9c","line":710,"updated":"2020-11-24 22:46:01.000000000","message":"so this \"no overlaps\" auditing isn\u0027t super sophisticated...\n\nno overlapping active is a pretty obvious win, that mostly applies to any other state (if we have some found or created - they shouldn\u0027t overlap)\n\nBut it does get hairy when a shrinking shard overlaps with an active - but I guess this check didn\u0027t catch that anyway!","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":783,"context_line":"            # shard is to shard itself"},{"line_number":784,"context_line":"            self.logger.debug(\u0027Updating %s shard_range(s) from root\u0027,"},{"line_number":785,"context_line":"                              len(shard_ranges))"},{"line_number":786,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":787,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":788,"context_line":""},{"line_number":789,"context_line":"        delete_age \u003d time.time() - self.reclaim_age"}],"source_content_type":"text/x-python","patch_set":12,"id":"4315446c_3efe60d0","line":786,"updated":"2020-11-24 22:46:01.000000000","message":"this is like the flux capacitor of root driven shrinking right here - love it!","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":1405,"context_line":"                # else, no errors, but no rows found either. keep going,"},{"line_number":1406,"context_line":"                # and don\u0027t count it against our batch size"},{"line_number":1407,"context_line":"            else:"},{"line_number":1408,"context_line":"                self.logger.info(\u0027Stopped cleave at unready %s\u0027, shard_range)"},{"line_number":1409,"context_line":"                break"},{"line_number":1410,"context_line":""},{"line_number":1411,"context_line":"        if not ranges_done:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3432acf0_7b917cfc","line":1408,"updated":"2020-11-24 22:46:01.000000000","message":"maybe someday there\u0027s a way to make this loop more obviously overlap aware.... maybe not without route finding code?\n\nwhich I guess is why the audits need to special case \"don\u0027t blow up on normal expected overlaps from shrinking\"","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":318,"context_line":"        self.ranges_done +\u003d 1"},{"line_number":319,"context_line":"        self.ranges_todo -\u003d 1"},{"line_number":320,"context_line":"        if new_cursor is not None:"},{"line_number":321,"context_line":"            self.cursor \u003d new_cursor"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"    def done(self):"},{"line_number":324,"context_line":"        return all((self.misplaced_done, self.cleaving_done,"}],"source_content_type":"text/x-python","patch_set":13,"id":"03674f2e_4249ab24","line":321,"updated":"2020-12-17 17:44:01.000000000","message":"quite reasonable","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":318,"context_line":"        self.ranges_done +\u003d 1"},{"line_number":319,"context_line":"        self.ranges_todo -\u003d 1"},{"line_number":320,"context_line":"        if new_cursor is not None:"},{"line_number":321,"context_line":"            self.cursor \u003d new_cursor"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"    def done(self):"},{"line_number":324,"context_line":"        return all((self.misplaced_done, self.cleaving_done,"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a1d9efa_2588596a","line":321,"in_reply_to":"03674f2e_4249ab24","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":427,"context_line":"                (internal_client_conf_path, err))"},{"line_number":428,"context_line":"        self.reported \u003d 0"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"    def _is_auto_shard(self, **kwargs):"},{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"}],"source_content_type":"text/x-python","patch_set":13,"id":"04ab3f5c_c83834e7","line":430,"updated":"2020-12-17 17:44:01.000000000","message":"is it weird that the \"recommended\" interface to \"should I auto shard\" is a private method that expects commandline kwargs","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":427,"context_line":"                (internal_client_conf_path, err))"},{"line_number":428,"context_line":"        self.reported \u003d 0"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"    def _is_auto_shard(self, **kwargs):"},{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"}],"source_content_type":"text/x-python","patch_set":13,"id":"7e3f7348_135945aa","line":430,"in_reply_to":"04ab3f5c_c83834e7","updated":"2020-12-18 11:28:26.000000000","message":"thanks for your patch - will squash\n\nI wasn\u0027t very happy with this kwarg passing, but also frustrated that there\u0027s no current pattern to override conf from the command line","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":430,"context_line":"    def _is_auto_shard(self, **kwargs):"},{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"},{"line_number":434,"context_line":"        return self.auto_shard"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"    def _zero_stats(self):"}],"source_content_type":"text/x-python","patch_set":13,"id":"84a3a895_6fa9dc8f","line":433,"updated":"2020-12-17 17:44:01.000000000","message":"it\u0027s a little weird for me that this boolean command line option is just a string; argparse supports the whole --auto-shard --no-auto-shard thing but I guess --auto-shard\u003dtrue is fine","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":430,"context_line":"    def _is_auto_shard(self, **kwargs):"},{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"},{"line_number":434,"context_line":"        return self.auto_shard"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"    def _zero_stats(self):"}],"source_content_type":"text/x-python","patch_set":13,"id":"ae67d614_8062db77","line":433,"in_reply_to":"84a3a895_6fa9dc8f","updated":"2020-12-18 11:28:26.000000000","message":"swift-container-sharder uses optparse still 😞\n\nbut...I hadn\u0027t thought of the --(no-)\u003coption\u003e pattern, I\u0027ll change swift-container-sharder to accept --no-auto-shard, which is all we need right now, and that\u0027ll fit if we ever move to argparse","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"},{"line_number":434,"context_line":"        return self.auto_shard"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"    def _zero_stats(self):"},{"line_number":437,"context_line":"        \"\"\"Zero out the stats.\"\"\""}],"source_content_type":"text/x-python","patch_set":13,"id":"c0ce00cc_4ac6face","line":434,"updated":"2020-12-17 17:44:01.000000000","message":"(but the attribute is public?)","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":431,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":432,"context_line":"        if auto_shard is not None:"},{"line_number":433,"context_line":"            return config_true_value(auto_shard)"},{"line_number":434,"context_line":"        return self.auto_shard"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"    def _zero_stats(self):"},{"line_number":437,"context_line":"        \"\"\"Zero out the stats.\"\"\""}],"source_content_type":"text/x-python","patch_set":13,"id":"09ec5e28_1e5f6175","line":434,"in_reply_to":"c0ce00cc_4ac6face","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":705,"context_line":""},{"line_number":706,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):"},{"line_number":707,"context_line":"            shard_ranges \u003d [sr for sr in broker.get_shard_ranges()"},{"line_number":708,"context_line":"                            if sr.state !\u003d ShardRange.SHRINKING]"},{"line_number":709,"context_line":"            missing_ranges \u003d find_missing_ranges(shard_ranges)"},{"line_number":710,"context_line":"            if missing_ranges:"},{"line_number":711,"context_line":"                warnings.append("}],"source_content_type":"text/x-python","patch_set":13,"id":"74065cd3_0e04bafc","line":708,"updated":"2020-12-17 17:44:01.000000000","message":"I guess SHRINKING is the only \"non-deleted\" state we need to filter?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":705,"context_line":""},{"line_number":706,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):"},{"line_number":707,"context_line":"            shard_ranges \u003d [sr for sr in broker.get_shard_ranges()"},{"line_number":708,"context_line":"                            if sr.state !\u003d ShardRange.SHRINKING]"},{"line_number":709,"context_line":"            missing_ranges \u003d find_missing_ranges(shard_ranges)"},{"line_number":710,"context_line":"            if missing_ranges:"},{"line_number":711,"context_line":"                warnings.append("}],"source_content_type":"text/x-python","patch_set":13,"id":"47d1ee5e_977546d1","line":708,"in_reply_to":"74065cd3_0e04bafc","updated":"2020-12-18 11:28:26.000000000","message":"yes, SHRUNK is also deleted, see _complete_sharding()","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":719,"context_line":"                # allow multiple shards in that state"},{"line_number":720,"context_line":"                continue"},{"line_number":721,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":722,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":723,"context_line":"            for overlapping_ranges in overlaps:"},{"line_number":724,"context_line":"                warnings.append("},{"line_number":725,"context_line":"                    \u0027overlapping ranges in state %s: %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":13,"id":"5ded8fe8_9b42e5aa","line":722,"updated":"2020-12-17 17:44:01.000000000","message":"I\u0027m guessing we don\u0027t care about overlapping SHRUNK states either - but for that state default kwargs probably comes back as an empty list because they\u0027re deleted?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":719,"context_line":"                # allow multiple shards in that state"},{"line_number":720,"context_line":"                continue"},{"line_number":721,"context_line":"            shard_ranges \u003d broker.get_shard_ranges(states\u003dstate)"},{"line_number":722,"context_line":"            overlaps \u003d find_overlapping_ranges(shard_ranges)"},{"line_number":723,"context_line":"            for overlapping_ranges in overlaps:"},{"line_number":724,"context_line":"                warnings.append("},{"line_number":725,"context_line":"                    \u0027overlapping ranges in state %s: %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":13,"id":"fb79a0d9_44bfaced","line":722,"in_reply_to":"5ded8fe8_9b42e5aa","updated":"2020-12-18 11:28:26.000000000","message":"yes, SHRUNK shards are always deleted in _complete_sharding()\n\noff-topic but related: I\u0027m not sure why we only check for overlaps in the *same* state...I think Tim has some improvements for these audit functions to address that","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":798,"context_line":"            orig_own_shard_range \u003d own_shard_range"},{"line_number":799,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":800,"context_line":"            if (orig_own_shard_range !\u003d own_shard_range or"},{"line_number":801,"context_line":"                    orig_own_shard_range.state !\u003d own_shard_range.state):"},{"line_number":802,"context_line":"                self.logger.debug("},{"line_number":803,"context_line":"                    \u0027Updated own shard range from %s to %s\u0027,"},{"line_number":804,"context_line":"                    orig_own_shard_range, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3364ffaf_9388a285","line":801,"updated":"2020-12-17 17:44:01.000000000","message":"it\u0027s weird for me that shard range equality only tests ranges.\n\nwhat if the range \u0026 state is the same buy the epoch got updated?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":798,"context_line":"            orig_own_shard_range \u003d own_shard_range"},{"line_number":799,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":800,"context_line":"            if (orig_own_shard_range !\u003d own_shard_range or"},{"line_number":801,"context_line":"                    orig_own_shard_range.state !\u003d own_shard_range.state):"},{"line_number":802,"context_line":"                self.logger.debug("},{"line_number":803,"context_line":"                    \u0027Updated own shard range from %s to %s\u0027,"},{"line_number":804,"context_line":"                    orig_own_shard_range, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":13,"id":"1725d627_b1c6ac32","line":801,"in_reply_to":"3364ffaf_9388a285","updated":"2020-12-18 11:28:26.000000000","message":"agree, I was surprised on coming back to this that ShardRange equality is based only on the bounds.\n\nIn this case it suited my intent of only logging when the bounds or state change, because those would be significant events in terms of the shard\u0027s behaviour.","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":803,"context_line":"                    \u0027Updated own shard range from %s to %s\u0027,"},{"line_number":804,"context_line":"                    orig_own_shard_range, own_shard_range)"},{"line_number":805,"context_line":"            if own_shard_range.state in (ShardRange.SHRINKING,"},{"line_number":806,"context_line":"                                         ShardRange.SHRUNK):"},{"line_number":807,"context_line":"                # If the up-to-date state is shrinking, save off *all* shards"},{"line_number":808,"context_line":"                # returned because these may contain shards into which this"},{"line_number":809,"context_line":"                # shard is to shrink itself; shrinking is the only case when we"}],"source_content_type":"text/x-python","patch_set":13,"id":"1cbcbb8f_193b5030","line":806,"updated":"2020-12-17 17:44:01.000000000","message":"older swifts moved ranges from shrinking to sharded - I don\u0027t really see the harm in letting those update to (i.e. if we managed to get a sharding range in us; why not let it update)","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":803,"context_line":"                    \u0027Updated own shard range from %s to %s\u0027,"},{"line_number":804,"context_line":"                    orig_own_shard_range, own_shard_range)"},{"line_number":805,"context_line":"            if own_shard_range.state in (ShardRange.SHRINKING,"},{"line_number":806,"context_line":"                                         ShardRange.SHRUNK):"},{"line_number":807,"context_line":"                # If the up-to-date state is shrinking, save off *all* shards"},{"line_number":808,"context_line":"                # returned because these may contain shards into which this"},{"line_number":809,"context_line":"                # shard is to shrink itself; shrinking is the only case when we"}],"source_content_type":"text/x-python","patch_set":13,"id":"5bbb9629_e6eb4156","line":806,"in_reply_to":"1cbcbb8f_193b5030","updated":"2020-12-18 11:28:26.000000000","message":"my concern is that a shard that is still in the process of sharding may have state ShardRange.SHARDED, and I would prefer to not have to reason about those shards getting (other) sub-shards from the root","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":1357,"context_line":"        self._min_stat(\u0027cleaved\u0027, \u0027min_time\u0027, elapsed)"},{"line_number":1358,"context_line":"        self._max_stat(\u0027cleaved\u0027, \u0027max_time\u0027, elapsed)"},{"line_number":1359,"context_line":"        broker.merge_shard_ranges(shard_range)"},{"line_number":1360,"context_line":"        cleaving_context.range_done(shard_range.upper_str)"},{"line_number":1361,"context_line":"        if shard_range.upper \u003e\u003d own_shard_range.upper:"},{"line_number":1362,"context_line":"            # cleaving complete"},{"line_number":1363,"context_line":"            cleaving_context.cleaving_done \u003d True"}],"source_content_type":"text/x-python","patch_set":13,"id":"2c03d688_cbb03bc6","line":1360,"updated":"2020-12-17 17:44:01.000000000","message":"nice little cleanup/consolidation - KUDOS","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":1357,"context_line":"        self._min_stat(\u0027cleaved\u0027, \u0027min_time\u0027, elapsed)"},{"line_number":1358,"context_line":"        self._max_stat(\u0027cleaved\u0027, \u0027max_time\u0027, elapsed)"},{"line_number":1359,"context_line":"        broker.merge_shard_ranges(shard_range)"},{"line_number":1360,"context_line":"        cleaving_context.range_done(shard_range.upper_str)"},{"line_number":1361,"context_line":"        if shard_range.upper \u003e\u003d own_shard_range.upper:"},{"line_number":1362,"context_line":"            # cleaving complete"},{"line_number":1363,"context_line":"            cleaving_context.cleaving_done \u003d True"}],"source_content_type":"text/x-python","patch_set":13,"id":"5d8c0009_64124a23","line":1360,"in_reply_to":"2c03d688_cbb03bc6","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":1399,"context_line":"        if cleaving_context.cursor:"},{"line_number":1400,"context_line":"            # always update ranges_todo in case more ranges have been found"},{"line_number":1401,"context_line":"            # since last visit"},{"line_number":1402,"context_line":"            cleaving_context.ranges_todo \u003d len(ranges_todo)"},{"line_number":1403,"context_line":"            self.logger.debug(\u0027Continuing to cleave (%s done, %s todo): %s\u0027,"},{"line_number":1404,"context_line":"                              cleaving_context.ranges_done,"},{"line_number":1405,"context_line":"                              cleaving_context.ranges_todo,"}],"source_content_type":"text/x-python","patch_set":13,"id":"8e029d7e_3acee933","line":1402,"updated":"2020-12-17 17:44:01.000000000","message":"little bit of forced bookkeeping","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":1399,"context_line":"        if cleaving_context.cursor:"},{"line_number":1400,"context_line":"            # always update ranges_todo in case more ranges have been found"},{"line_number":1401,"context_line":"            # since last visit"},{"line_number":1402,"context_line":"            cleaving_context.ranges_todo \u003d len(ranges_todo)"},{"line_number":1403,"context_line":"            self.logger.debug(\u0027Continuing to cleave (%s done, %s todo): %s\u0027,"},{"line_number":1404,"context_line":"                              cleaving_context.ranges_done,"},{"line_number":1405,"context_line":"                              cleaving_context.ranges_todo,"}],"source_content_type":"text/x-python","patch_set":13,"id":"b6247c09_5ea67a7f","line":1402,"in_reply_to":"8e029d7e_3acee933","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":1419,"context_line":"                # shard range) are not normally expected in a shard but can"},{"line_number":1420,"context_line":"                # occur if there is an overlapping shard range that has been"},{"line_number":1421,"context_line":"                # discovered from the root."},{"line_number":1422,"context_line":"                cleaving_context.range_done(None)  # don\u0027t move the cursor"},{"line_number":1423,"context_line":"                continue"},{"line_number":1424,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1425,"context_line":"                                       ShardRange.CLEAVED,"}],"source_content_type":"text/x-python","patch_set":13,"id":"bbdfbf30_e7808049","line":1422,"updated":"2020-12-17 17:44:01.000000000","message":"this is mostly to keep bookkeeping uptodate - i\u0027m not sure it\u0027s 100% necessary (but skipping over shrinking states seems reasonable!)","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":1419,"context_line":"                # shard range) are not normally expected in a shard but can"},{"line_number":1420,"context_line":"                # occur if there is an overlapping shard range that has been"},{"line_number":1421,"context_line":"                # discovered from the root."},{"line_number":1422,"context_line":"                cleaving_context.range_done(None)  # don\u0027t move the cursor"},{"line_number":1423,"context_line":"                continue"},{"line_number":1424,"context_line":"            elif shard_range.state in (ShardRange.CREATED,"},{"line_number":1425,"context_line":"                                       ShardRange.CLEAVED,"}],"source_content_type":"text/x-python","patch_set":13,"id":"b3738c1e_049bff3a","line":1422,"in_reply_to":"bbdfbf30_e7808049","updated":"2020-12-18 11:28:26.000000000","message":"yes, the logging of ranges to do/done was a little confusing before, when shrinking was skipped","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":1565,"context_line":"            # have new objects sitting in them that may need to move."},{"line_number":1566,"context_line":"            return"},{"line_number":1567,"context_line":""},{"line_number":1568,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self._is_auto_shard(**kwargs)"},{"line_number":1569,"context_line":"        if state in (UNSHARDED, COLLAPSED):"},{"line_number":1570,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":1571,"context_line":"                # bootstrap sharding of root container"}],"source_content_type":"text/x-python","patch_set":13,"id":"6a3012ae_5e199044","line":1568,"updated":"2020-12-17 17:44:01.000000000","message":"it looks like **kwargs plumbing is for this","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":1565,"context_line":"            # have new objects sitting in them that may need to move."},{"line_number":1566,"context_line":"            return"},{"line_number":1567,"context_line":""},{"line_number":1568,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self._is_auto_shard(**kwargs)"},{"line_number":1569,"context_line":"        if state in (UNSHARDED, COLLAPSED):"},{"line_number":1570,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":1571,"context_line":"                # bootstrap sharding of root container"}],"source_content_type":"text/x-python","patch_set":13,"id":"bd23ccce_77fbcc7a","line":1568,"in_reply_to":"6a3012ae_5e199044","updated":"2020-12-18 11:28:26.000000000","message":"Done","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":1746,"context_line":"        begin \u003d self.reported \u003d time.time()"},{"line_number":1747,"context_line":"        self._one_shard_cycle(devices_to_shard\u003ddevices_to_shard,"},{"line_number":1748,"context_line":"                              partitions_to_shard\u003dpartitions_to_shard,"},{"line_number":1749,"context_line":"                              **kwargs)"},{"line_number":1750,"context_line":"        elapsed \u003d time.time() - begin"},{"line_number":1751,"context_line":"        self.logger.info("},{"line_number":1752,"context_line":"            \u0027Container sharder \"once\" mode completed: %.02fs\u0027, elapsed)"}],"source_content_type":"text/x-python","patch_set":13,"id":"43ac6956_71d6181d","line":1749,"updated":"2020-12-17 17:44:01.000000000","message":"alternatively set self.auto_shard form the kwarg here?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":1746,"context_line":"        begin \u003d self.reported \u003d time.time()"},{"line_number":1747,"context_line":"        self._one_shard_cycle(devices_to_shard\u003ddevices_to_shard,"},{"line_number":1748,"context_line":"                              partitions_to_shard\u003dpartitions_to_shard,"},{"line_number":1749,"context_line":"                              **kwargs)"},{"line_number":1750,"context_line":"        elapsed \u003d time.time() - begin"},{"line_number":1751,"context_line":"        self.logger.info("},{"line_number":1752,"context_line":"            \u0027Container sharder \"once\" mode completed: %.02fs\u0027, elapsed)"}],"source_content_type":"text/x-python","patch_set":13,"id":"6d0176a7_2d63489d","line":1749,"in_reply_to":"43ac6956_71d6181d","updated":"2020-12-18 11:28:26.000000000","message":"I was trying to avoid permanently changing the state of the daemon instance as a side effect of calling a run_ method. Ideally, I\u0027d like to pass the cli options to the daemon constructor, but that means changing the Daemon interface.\n\nKnown usage is that daemons instances are used just once, so we could modify the conf here, but it feels \u0027naughty\u0027 in principle for one call to run_once to modify the behaviour for subsequent calls.","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":false,"context_lines":[{"line_number":1746,"context_line":"        begin \u003d self.reported \u003d time.time()"},{"line_number":1747,"context_line":"        self._one_shard_cycle(devices_to_shard\u003ddevices_to_shard,"},{"line_number":1748,"context_line":"                              partitions_to_shard\u003dpartitions_to_shard,"},{"line_number":1749,"context_line":"                              **kwargs)"},{"line_number":1750,"context_line":"        elapsed \u003d time.time() - begin"},{"line_number":1751,"context_line":"        self.logger.info("},{"line_number":1752,"context_line":"            \u0027Container sharder \"once\" mode completed: %.02fs\u0027, elapsed)"}],"source_content_type":"text/x-python","patch_set":13,"id":"81468dc2_2e2175b8","line":1749,"in_reply_to":"6d0176a7_2d63489d","updated":"2020-12-18 16:36:41.000000000","message":"I think we\u0027re thinking about this broadly the same - for some options like parts/devices there\u0027s no configuration counterpart - but for options mirroed in the config, it\u0027s just like a commandline override!\n\nFWIW I don\u0027t see anything sinister about a command line argument permeneantly overrrideing the behavior of the daemon for the lifetime of the process.  It seems like you\u0027re considering the lifecycle even more broadly if some orchastrator was calling run_once directly with **{\u0027auto_shard\u0027: False}, and then **{}?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":true,"context_lines":[{"line_number":1715,"context_line":"    def _set_auto_shard_from_command_line(self, **kwargs):"},{"line_number":1716,"context_line":"        auto_shard \u003d kwargs.get(\u0027auto_shard\u0027, None)"},{"line_number":1717,"context_line":"        if auto_shard is not None:"},{"line_number":1718,"context_line":"            self.auto_shard \u003d config_true_value(auto_shard)"},{"line_number":1719,"context_line":""},{"line_number":1720,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":1721,"context_line":"        \"\"\"Run the container sharder until stopped.\"\"\""}],"source_content_type":"text/x-python","patch_set":14,"id":"167c8b9b_41a477b0","line":1718,"updated":"2020-12-18 16:36:41.000000000","message":"given the change in the bin script: I think auto_shard will either be None or False but never truthy?","commit_id":"d277960161119face2eb9d617822b2645f19b2c1"}],"test/probe/test_sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":2425,"context_line":"        additional_args \u003d kwargs.get(\u0027additional_args\u0027, [])"},{"line_number":2426,"context_line":"        if not isinstance(additional_args, list):"},{"line_number":2427,"context_line":"            additional_args \u003d [additional_args]"},{"line_number":2428,"context_line":"        additional_args.append(\u0027--auto-shard\u003dfalse\u0027)"},{"line_number":2429,"context_line":"        kwargs[\u0027additional_args\u0027] \u003d additional_args"},{"line_number":2430,"context_line":"        self.sharders.once(**kwargs)"},{"line_number":2431,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"007e9402_6ca292ad","line":2428,"updated":"2020-12-17 17:44:01.000000000","message":"makes sense that the \"Managed\" sharding test always does false!","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":2425,"context_line":"        additional_args \u003d kwargs.get(\u0027additional_args\u0027, [])"},{"line_number":2426,"context_line":"        if not isinstance(additional_args, list):"},{"line_number":2427,"context_line":"            additional_args \u003d [additional_args]"},{"line_number":2428,"context_line":"        additional_args.append(\u0027--auto-shard\u003dfalse\u0027)"},{"line_number":2429,"context_line":"        kwargs[\u0027additional_args\u0027] \u003d additional_args"},{"line_number":2430,"context_line":"        self.sharders.once(**kwargs)"},{"line_number":2431,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"3e6e6aa9_12709b13","line":2428,"in_reply_to":"007e9402_6ca292ad","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"}],"test/unit/cli/test_manage_shard_ranges.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d47f6073ce5c2665a8684436063baf3be5d197cd","unresolved":false,"context_lines":[{"line_number":472,"context_line":"            in_.write(\u0027Y\\n\u0027)  # do it!!"},{"line_number":473,"context_line":"            in_.seek(0)"},{"line_number":474,"context_line":"            main([broker.db_file, \u0027shrink\u0027, \u0027.shards_a/c-xxx-2\u0027,"},{"line_number":475,"context_line":"                  \u0027--shrink-size\u003d50\u0027, \u0027--acceptor\u003d.shards_a/c-xxx-1\u0027])"},{"line_number":476,"context_line":"        out_lines \u003d out.getvalue().splitlines()"},{"line_number":477,"context_line":""},{"line_number":478,"context_line":"        self.assertEqual(\u0027Donor .shards_a/c-xxx-2\u0027, out_lines[0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_2440cd56","line":475,"updated":"2020-07-29 03:58:46.000000000","message":"As I mentioned before, what happens when you specify an acceptor that isn\u0027t next to the shard, ie:\n\n  main([broker.db_file, \u0027shrink\u0027, \u0027.shards_a/c-xxx-1\u0027, \u0027--shrink-size\u003d110\u0027, \u0027--merge-size 1200\u0027, \u0027--acceptor\u003d.shards_a/c-xxx-3\u0027])\n\nWill it then go create a shard 3 that is a - g and still have shard 2 that\u0027s c - e?","commit_id":"79714186abaa9bf023653b7e501c8d6ba6b9e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8002ba481c3ba40a884c8677e75d69c7fafa885b","unresolved":false,"context_lines":[{"line_number":482,"context_line":"        self.assertEqual(\u0027.shards_a/c-xxx-2\u0027, new_shard_ranges[1].name)"},{"line_number":483,"context_line":"        self.assertEqual(ShardRange.SHRINKING, new_shard_ranges[1].state)"},{"line_number":484,"context_line":"        self.assertEqual(\u0027.shards_a/c-xxx-1\u0027, new_shard_ranges[0].name)"},{"line_number":485,"context_line":"        self.assertEqual(\u0027e\u0027, new_shard_ranges[0].upper)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_f9b91ce9","line":485,"updated":"2020-07-31 18:21:52.000000000","message":"basically none of these are correct yet","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"}],"test/unit/container/test_backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":3982,"context_line":"            ShardRange(\u0027.shards_a/c1\u0027, next(self.ts), \u0027d\u0027, \u0027m\u0027,"},{"line_number":3983,"context_line":"                       state\u003dShardRange.SHARDING),"},{"line_number":3984,"context_line":"            ShardRange(\u0027.shards_a/c1_2\u0027, next(self.ts), \u0027j\u0027, \u0027m\u0027,"},{"line_number":3985,"context_line":"                       state\u003dShardRange.CREATED),"},{"line_number":3986,"context_line":"            ShardRange(\u0027.shards_a/c2\u0027, next(self.ts), \u0027m\u0027, \u0027\u0027,"},{"line_number":3987,"context_line":"                       state\u003dShardRange.ACTIVE),"},{"line_number":3988,"context_line":"        ]"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_92d25bff","line":3985,"updated":"2020-11-10 00:30:21.000000000","message":"Shard ordering fix; check.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4005,"context_line":"        self.assertEqual([shard_ranges[2]], actual)"},{"line_number":4006,"context_line":"        actual \u003d broker.get_shard_ranges(states\u003dSHARD_UPDATE_STATES,"},{"line_number":4007,"context_line":"                                         includes\u003d\u0027k\u0027)"},{"line_number":4008,"context_line":"        self.assertEqual([shard_ranges[3]], actual)"},{"line_number":4009,"context_line":""},{"line_number":4010,"context_line":"    @with_tempdir"},{"line_number":4011,"context_line":"    def test_get_shard_ranges_with_shrinking_overlaps(self, tempdir):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_e0b1dd45","line":4008,"updated":"2020-11-17 14:50:57.000000000","message":"not sure this is what we want: \u0027k\u0027 is in the sub-shard c1_2 and that is where the update would have previously gone but know the update will got to the sharding shard c1.\n\nI re-wrote the test in follow-on to make it clearer","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"}],"test/unit/container/test_replicator.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba5b69ba418a0a7bdf0de57f196054a859ce1025","unresolved":false,"context_lines":[{"line_number":1493,"context_line":"        shard_ranges[1].update_meta(13, 123)"},{"line_number":1494,"context_line":"        broker.merge_shard_ranges(shard_ranges[1])"},{"line_number":1495,"context_line":"        broker_ranges \u003d broker.get_all_shard_range_data()"},{"line_number":1496,"context_line":"        expected_ranges \u003d shard_ranges[:-1] + [own_sr] + shard_ranges[-1:]"},{"line_number":1497,"context_line":"        self.assertShardRangesEqual(expected_ranges, broker_ranges)"},{"line_number":1498,"context_line":""},{"line_number":1499,"context_line":"        def check_stats(daemon):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_04755b32","line":1496,"updated":"2020-11-10 23:05:51.000000000","message":"Why do we suddenly have to insert the own shard range just before the end. Could we just add a short to assertShardRangesEqual or something to make it easy to use?","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c428f305973e2788acbd804562f6134d3310ce19","unresolved":false,"context_lines":[{"line_number":5081,"context_line":"        shard_ranges \u003d self._stabilize(shard_ranges)"},{"line_number":5082,"context_line":""},{"line_number":5083,"context_line":"        acceptors \u003d find_shinrking_acceptors(shard_ranges[0], shard_ranges)"},{"line_number":5084,"context_line":"        self.assertEqual(acceptors, [shard_ranges[1]])"},{"line_number":5085,"context_line":""},{"line_number":5086,"context_line":"    def test_shadowed_overlap_shrinks_left(self):"},{"line_number":5087,"context_line":"        epoch1 \u003d next(self.ts)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_cf28ca87","line":5084,"updated":"2020-07-31 19:58:47.000000000","message":"how different is this from the case you were asking where the small first range shirks to the right?","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8002ba481c3ba40a884c8677e75d69c7fafa885b","unresolved":false,"context_lines":[{"line_number":5264,"context_line":"        acceptors \u003d find_shinrking_acceptors(donor, shard_ranges)"},{"line_number":5265,"context_line":"        expected \u003d [sr for sr in shard_ranges"},{"line_number":5266,"context_line":"                    if sr.name in (self.srn(epoch3, i) for i in (1, 2))]"},{"line_number":5267,"context_line":"        self.assertEqual(expected, acceptors)"},{"line_number":5268,"context_line":""},{"line_number":5269,"context_line":""},{"line_number":5270,"context_line":"class TestCleavingContext(BaseTestSharder):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_59d00820","line":5267,"updated":"2020-07-31 18:21:52.000000000","message":"I want another test that has *3* acceptors","commit_id":"396a96ea7fec8d9a32a0a78bab929c67864fded5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6533276fa2b11251216e0227079dd6b6cfcce470","unresolved":false,"context_lines":[{"line_number":5023,"context_line":"        broker.initialize()"},{"line_number":5024,"context_line":"        broker.merge_shard_ranges(shard_ranges)"},{"line_number":5025,"context_line":"        stable_order \u003d broker.get_shard_ranges()"},{"line_number":5026,"context_line":"        # ideally tests literals are ordered"},{"line_number":5027,"context_line":"        self.assertEqual(shard_ranges, stable_order)"},{"line_number":5028,"context_line":"        return stable_order"},{"line_number":5029,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f65232a_629e2ab9","line":5026,"updated":"2020-10-26 18:55:59.000000000","message":"I think it is a requirement that they are for find_shrinking_acceptors to behave as expected?","commit_id":"c06fc4f85d97acb9f77682f03866e4b1ec94c395"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"04bdc80e8c12ae2898b05e088f8d88b96815ec95","unresolved":false,"context_lines":[{"line_number":4561,"context_line":"        self.assertIn(\u0027Audit warnings for shard %s\u0027 % broker.db_file, lines[0])"},{"line_number":4562,"context_line":"        self.assertNotIn(\u0027account not in shards namespace\u0027, lines[0])"},{"line_number":4563,"context_line":"        self.assertNotIn(\u0027missing own shard range\u0027, lines[0])"},{"line_number":4564,"context_line":"        self.assertIn(\u0027root has no matching shard range\u0027, lines[0])"},{"line_number":4565,"context_line":"        self.assertNotIn(\u0027unable to get shard ranges from root\u0027, lines[0])"},{"line_number":4566,"context_line":"        self._assert_stats(expected_stats, sharder, \u0027audit_shard\u0027)"},{"line_number":4567,"context_line":"        self.assertFalse(lines[1:])"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_f7be21b2","side":"PARENT","line":4564,"updated":"2020-11-11 14:39:23.000000000","message":"i guess we just eat the new shard boundaries from the root","commit_id":"2b79d0befd094da1ae4c11aea9d56993fb9e1003"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":4363,"context_line":"        # Put the shards back to a \"useful\" state"},{"line_number":4364,"context_line":"        shard_ranges \u003d self._make_shard_ranges(shard_bounds,"},{"line_number":4365,"context_line":"                                               ShardRange.ACTIVE)"},{"line_number":4366,"context_line":"        broker.merge_shard_ranges(shard_ranges)"},{"line_number":4367,"context_line":""},{"line_number":4368,"context_line":"        def assert_missing_warning(line):"},{"line_number":4369,"context_line":"            self.assertIn("}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_b24317af","line":4366,"updated":"2020-11-10 00:30:21.000000000","message":"\u0027K -- overlapping shards don\u0027t cause audit failures, check.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":4382,"context_line":"                    sharder._audit_container(broker)"},{"line_number":4383,"context_line":"            lines \u003d sharder.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":4384,"context_line":"            assert_missing_warning(lines[0])"},{"line_number":4385,"context_line":"            assert_overlap_warning(lines[0], \u0027active\u0027)"},{"line_number":4386,"context_line":"            self.assertFalse(lines[1:])"},{"line_number":4387,"context_line":"            self.assertFalse(sharder.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4388,"context_line":"            self._assert_stats(expected_stats, sharder, \u0027audit_root\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_32a2674c","line":4385,"updated":"2020-11-10 00:30:21.000000000","message":"This part\u0027s a little weird; I guess we were previously relying on dictionary item ordering -- and somehow it just all magically worked out before!?","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3a3fb9507d5fa46b0b3167342edc1d85a1da997","unresolved":false,"context_lines":[{"line_number":4573,"context_line":"        broker.merge_shard_ranges([own_shard_range])"},{"line_number":4574,"context_line":"        sharder, mock_swift \u003d self.call_audit_container(broker, shard_ranges)"},{"line_number":4575,"context_line":"        self._assert_stats(expected_stats, sharder, \u0027audit_shard\u0027)"},{"line_number":4576,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027warning\u0027))"},{"line_number":4577,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4578,"context_line":"        self.assertFalse(broker.is_deleted())"},{"line_number":4579,"context_line":"        expected_headers \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f621f24_f2690ff9","line":4576,"updated":"2020-11-10 00:30:21.000000000","message":"Right, and these last two demonstrate how we\u0027ll find our own shard even when our bounds don\u0027t match what the root reports for us. Should probably assert that our bounds changed afterwards, too.","commit_id":"d45a674004b109c840feacae4f640101d397a141"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4333,"context_line":"        shard_bounds \u003d ((\u0027a\u0027, \u0027j\u0027), (\u0027k\u0027, \u0027t\u0027), (\u0027s\u0027, \u0027z\u0027))"},{"line_number":4334,"context_line":"        for state, state_text in ShardRange.STATES.items():"},{"line_number":4335,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":4336,"context_line":"                continue  # tested separately below"},{"line_number":4337,"context_line":"            shard_ranges \u003d self._make_shard_ranges(shard_bounds, state)"},{"line_number":4338,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":4339,"context_line":"            with self._mock_sharder() as sharder:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_1b90cd94","line":4336,"updated":"2020-11-17 14:50:57.000000000","message":"+1","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4363,"context_line":"        # Put the shards back to a \"useful\" state"},{"line_number":4364,"context_line":"        shard_ranges \u003d self._make_shard_ranges(shard_bounds,"},{"line_number":4365,"context_line":"                                               ShardRange.ACTIVE)"},{"line_number":4366,"context_line":"        broker.merge_shard_ranges(shard_ranges)"},{"line_number":4367,"context_line":""},{"line_number":4368,"context_line":"        def assert_missing_warning(line):"},{"line_number":4369,"context_line":"            self.assertIn("}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_5beb6511","line":4366,"updated":"2020-11-17 14:50:57.000000000","message":"+1","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4382,"context_line":"                    sharder._audit_container(broker)"},{"line_number":4383,"context_line":"            lines \u003d sharder.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":4384,"context_line":"            assert_missing_warning(lines[0])"},{"line_number":4385,"context_line":"            assert_overlap_warning(lines[0], \u0027active\u0027)"},{"line_number":4386,"context_line":"            self.assertFalse(lines[1:])"},{"line_number":4387,"context_line":"            self.assertFalse(sharder.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4388,"context_line":"            self._assert_stats(expected_stats, sharder, \u0027audit_root\u0027)"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_7bf461f1","line":4385,"updated":"2020-11-17 14:50:57.000000000","message":"+1, this is the state that all shards are in, so make it explicit","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4427,"context_line":"        broker.set_sharding_sysmeta(\u0027Root\u0027, \u0027a/c\u0027)"},{"line_number":4428,"context_line":"        # include overlaps to verify correct match for updating own shard range"},{"line_number":4429,"context_line":"        shard_bounds \u003d ("},{"line_number":4430,"context_line":"            (\u0027a\u0027, \u0027j\u0027), (\u0027k\u0027, \u0027t\u0027), (\u0027k\u0027, \u0027s\u0027), (\u0027l\u0027, \u0027s\u0027), (\u0027s\u0027, \u0027z\u0027))"},{"line_number":4431,"context_line":"        shard_ranges \u003d self._make_shard_ranges(shard_bounds, ShardRange.ACTIVE)"},{"line_number":4432,"context_line":"        shard_ranges[1].name \u003d broker.path"},{"line_number":4433,"context_line":"        expected_stats \u003d {\u0027attempted\u0027: 1, \u0027success\u0027: 0, \u0027failure\u0027: 1}"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_8a0c8894","line":4430,"updated":"2020-11-17 14:50:57.000000000","message":"these shard ranges won\u0027t all merge into a db because _make_shard_ranges gives the same name to k-s and l-s","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d4b697ff30aae90f5eef5242834b44299b153b0e","unresolved":false,"context_lines":[{"line_number":4573,"context_line":"        broker.merge_shard_ranges([own_shard_range])"},{"line_number":4574,"context_line":"        sharder, mock_swift \u003d self.call_audit_container(broker, shard_ranges)"},{"line_number":4575,"context_line":"        self._assert_stats(expected_stats, sharder, \u0027audit_shard\u0027)"},{"line_number":4576,"context_line":"        self.assertEqual([\u0027Updating 5 shard_range(s) from root\u0027],"},{"line_number":4577,"context_line":"                         sharder.logger.get_lines_for_level(\u0027debug\u0027))"},{"line_number":4578,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027warning\u0027))"},{"line_number":4579,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027error\u0027))"}],"source_content_type":"text/x-python","patch_set":11,"id":"1f621f24_8a0da8e9","line":4576,"updated":"2020-11-17 14:50:57.000000000","message":"this is a little artificial - the test mocks root returning 5 shard ranges but in reality the request to root is parameterised with marker/end-marker \u003d lower/upper so the number of shard ranges that root would return would be lower","commit_id":"219e8c20a2548ed7e2c0e81c1babfabd8582ae9f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":4355,"context_line":"                    sharder, \u0027_audit_shard_container\u0027) as mocked:"},{"line_number":4356,"context_line":"                sharder._audit_container(broker)"},{"line_number":4357,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027warning\u0027))"},{"line_number":4358,"context_line":"        self.assertFalse(sharder.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4359,"context_line":"        self._assert_stats({\u0027attempted\u0027: 1, \u0027success\u0027: 1, \u0027failure\u0027: 0},"},{"line_number":4360,"context_line":"                           sharder, \u0027audit_root\u0027)"},{"line_number":4361,"context_line":"        mocked.assert_not_called()"}],"source_content_type":"text/x-python","patch_set":12,"id":"2b7db287_94d5f1de","line":4358,"updated":"2020-11-24 22:46:01.000000000","message":"overlapping shrinkins is no problem; love it\n\nDo we have to reset the logger somewhere?","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":4370,"context_line":"                \u0027Audit failed for root %s\u0027 % broker.db_file, line)"},{"line_number":4371,"context_line":"            self.assertIn(\u0027missing range(s): -a j-k z-\u0027, line)"},{"line_number":4372,"context_line":""},{"line_number":4373,"context_line":"        def check_missing():"},{"line_number":4374,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":4375,"context_line":"            states \u003d (ShardRange.SHARDING, ShardRange.SHARDED)"},{"line_number":4376,"context_line":"            for state in states:"}],"source_content_type":"text/x-python","patch_set":12,"id":"5103b8c4_b8b3ba8a","line":4373,"updated":"2020-11-24 22:46:01.000000000","message":"cheeky little test closure","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":4396,"context_line":"        missing_shard_bounds \u003d ((\u0027\u0027, \u0027a\u0027), (\u0027j\u0027, \u0027k\u0027), (\u0027z\u0027, \u0027\u0027))"},{"line_number":4397,"context_line":"        shrinking_shard_ranges \u003d self._make_shard_ranges(missing_shard_bounds,"},{"line_number":4398,"context_line":"                                                         ShardRange.SHRINKING)"},{"line_number":4399,"context_line":"        broker.merge_shard_ranges(shrinking_shard_ranges)"},{"line_number":4400,"context_line":"        check_missing()"},{"line_number":4401,"context_line":""},{"line_number":4402,"context_line":"    def call_audit_container(self, broker, shard_ranges, exc\u003dNone):"}],"source_content_type":"text/x-python","patch_set":12,"id":"ca4da177_61ff8cca","line":4399,"updated":"2020-11-24 22:46:01.000000000","message":"i think this is just *adding* some new overlapping shard ranges (which we would assume are shrinking into *us*)","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"0b4a3371789a115fdf2a3258e060c051ec64fd7d","unresolved":true,"context_lines":[{"line_number":4396,"context_line":"        missing_shard_bounds \u003d ((\u0027\u0027, \u0027a\u0027), (\u0027j\u0027, \u0027k\u0027), (\u0027z\u0027, \u0027\u0027))"},{"line_number":4397,"context_line":"        shrinking_shard_ranges \u003d self._make_shard_ranges(missing_shard_bounds,"},{"line_number":4398,"context_line":"                                                         ShardRange.SHRINKING)"},{"line_number":4399,"context_line":"        broker.merge_shard_ranges(shrinking_shard_ranges)"},{"line_number":4400,"context_line":"        check_missing()"},{"line_number":4401,"context_line":""},{"line_number":4402,"context_line":"    def call_audit_container(self, broker, shard_ranges, exc\u003dNone):"}],"source_content_type":"text/x-python","patch_set":12,"id":"87b1d184_f611cae0","line":4399,"in_reply_to":"ca4da177_61ff8cca","updated":"2020-11-25 13:18:40.000000000","message":"the existing test checked that gaps (\u0027mising\u0027) were reported by audit; the intent of this test change was to verify that they are still reported even when covered by shrinking shards","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5028f0138003cadde223aa0c401327de3220ad75","unresolved":true,"context_lines":[{"line_number":4445,"context_line":"        shard_ranges.extend(self._make_shard_ranges(shard_bounds[3:4],"},{"line_number":4446,"context_line":"                                                    ShardRange.FOUND))"},{"line_number":4447,"context_line":"        shard_ranges.extend(self._make_shard_ranges(shard_bounds[4:],"},{"line_number":4448,"context_line":"                                                    ShardRange.CREATED))"},{"line_number":4449,"context_line":"        expected_stats \u003d {\u0027attempted\u0027: 1, \u0027success\u0027: 0, \u0027failure\u0027: 1}"},{"line_number":4450,"context_line":""},{"line_number":4451,"context_line":"        # bad account name"}],"source_content_type":"text/x-python","patch_set":12,"id":"61eb735d_009d10f0","line":4448,"updated":"2020-11-24 22:46:01.000000000","message":"maybe _make_shard_ranges could take a state_iter:\n\nactive, active, found, found, created","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cc8a55dd4533475441553ecb4e14a6adf9a7b890","unresolved":true,"context_lines":[{"line_number":4445,"context_line":"        shard_ranges.extend(self._make_shard_ranges(shard_bounds[3:4],"},{"line_number":4446,"context_line":"                                                    ShardRange.FOUND))"},{"line_number":4447,"context_line":"        shard_ranges.extend(self._make_shard_ranges(shard_bounds[4:],"},{"line_number":4448,"context_line":"                                                    ShardRange.CREATED))"},{"line_number":4449,"context_line":"        expected_stats \u003d {\u0027attempted\u0027: 1, \u0027success\u0027: 0, \u0027failure\u0027: 1}"},{"line_number":4450,"context_line":""},{"line_number":4451,"context_line":"        # bad account name"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f16d936_2a0bd053","line":4448,"in_reply_to":"61eb735d_009d10f0","updated":"2020-12-01 15:24:16.000000000","message":"done in follow up patch","commit_id":"9d428034576525cc22717ae6dcce81095f587b05"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":2264,"context_line":"        self.assertTrue(context.misplaced_done)"},{"line_number":2265,"context_line":"        self.assertFalse(context.cleaving_done)"},{"line_number":2266,"context_line":"        self.assertEqual(shard_ranges[2].upper_str, context.cursor)"},{"line_number":2267,"context_line":"        self.assertEqual(3, context.ranges_done)"},{"line_number":2268,"context_line":""},{"line_number":2269,"context_line":"        # run cleave - stops at shard range in FOUND state"},{"line_number":2270,"context_line":"        with self._mock_sharder() as sharder:"}],"source_content_type":"text/x-python","patch_set":13,"id":"f72a0366_e843e8f6","line":2267,"updated":"2020-12-17 17:44:01.000000000","message":"useful to assert on todo as well?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":2264,"context_line":"        self.assertTrue(context.misplaced_done)"},{"line_number":2265,"context_line":"        self.assertFalse(context.cleaving_done)"},{"line_number":2266,"context_line":"        self.assertEqual(shard_ranges[2].upper_str, context.cursor)"},{"line_number":2267,"context_line":"        self.assertEqual(3, context.ranges_done)"},{"line_number":2268,"context_line":""},{"line_number":2269,"context_line":"        # run cleave - stops at shard range in FOUND state"},{"line_number":2270,"context_line":"        with self._mock_sharder() as sharder:"}],"source_content_type":"text/x-python","patch_set":13,"id":"fb2aa5b6_bf7f8673","line":2267,"in_reply_to":"f72a0366_e843e8f6","updated":"2020-12-18 11:28:26.000000000","message":"Done","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":4400,"context_line":"        for state, state_text in ShardRange.STATES.items():"},{"line_number":4401,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":4402,"context_line":"                continue  # tested separately below"},{"line_number":4403,"context_line":"            shard_ranges \u003d self._make_shard_ranges(shard_bounds, state)"},{"line_number":4404,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":4405,"context_line":"            with self._mock_sharder() as sharder:"},{"line_number":4406,"context_line":"                with mock.patch.object("}],"source_content_type":"text/x-python","patch_set":13,"id":"46c194b7_c7ab8c13","line":4403,"updated":"2020-12-17 17:44:01.000000000","message":"these shard_ranges are deleted\u003d0, right?  does this test make sense for the sharded and shrunk states?","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":4400,"context_line":"        for state, state_text in ShardRange.STATES.items():"},{"line_number":4401,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":4402,"context_line":"                continue  # tested separately below"},{"line_number":4403,"context_line":"            shard_ranges \u003d self._make_shard_ranges(shard_bounds, state)"},{"line_number":4404,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":4405,"context_line":"            with self._mock_sharder() as sharder:"},{"line_number":4406,"context_line":"                with mock.patch.object("}],"source_content_type":"text/x-python","patch_set":13,"id":"6070f619_96c99ba3","line":4403,"in_reply_to":"46c194b7_c7ab8c13","updated":"2020-12-18 11:28:26.000000000","message":"good point","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":false,"context_lines":[{"line_number":4400,"context_line":"        for state, state_text in ShardRange.STATES.items():"},{"line_number":4401,"context_line":"            if state \u003d\u003d ShardRange.SHRINKING:"},{"line_number":4402,"context_line":"                continue  # tested separately below"},{"line_number":4403,"context_line":"            shard_ranges \u003d self._make_shard_ranges(shard_bounds, state)"},{"line_number":4404,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":4405,"context_line":"            with self._mock_sharder() as sharder:"},{"line_number":4406,"context_line":"                with mock.patch.object("}],"source_content_type":"text/x-python","patch_set":13,"id":"c9a61448_75f4e2cb","line":4403,"in_reply_to":"6070f619_96c99ba3","updated":"2020-12-18 16:36:41.000000000","message":"these updates are great!","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1113c57148e0b812aa1d3caee4c1c0ddb8b96108","unresolved":true,"context_lines":[{"line_number":5554,"context_line":"        ctx.range_done(None)"},{"line_number":5555,"context_line":"        self.assertEqual(2, ctx.ranges_done)"},{"line_number":5556,"context_line":"        self.assertEqual(3, ctx.ranges_todo)"},{"line_number":5557,"context_line":"        self.assertEqual(\u0027b\u0027, ctx.cursor)"},{"line_number":5558,"context_line":""},{"line_number":5559,"context_line":"        ctx.ranges_todo \u003d 9"},{"line_number":5560,"context_line":"        ctx.range_done(\u0027c\u0027)"}],"source_content_type":"text/x-python","patch_set":13,"id":"c665ce4c_f8b01310","line":5557,"updated":"2020-12-17 17:44:01.000000000","message":"this is a nice clean test assertion","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fbd049c67030788e2ab21cce91280b51d7920903","unresolved":false,"context_lines":[{"line_number":5554,"context_line":"        ctx.range_done(None)"},{"line_number":5555,"context_line":"        self.assertEqual(2, ctx.ranges_done)"},{"line_number":5556,"context_line":"        self.assertEqual(3, ctx.ranges_todo)"},{"line_number":5557,"context_line":"        self.assertEqual(\u0027b\u0027, ctx.cursor)"},{"line_number":5558,"context_line":""},{"line_number":5559,"context_line":"        ctx.ranges_todo \u003d 9"},{"line_number":5560,"context_line":"        ctx.range_done(\u0027c\u0027)"}],"source_content_type":"text/x-python","patch_set":13,"id":"57ba39a1_865aaa3a","line":5557,"in_reply_to":"c665ce4c_f8b01310","updated":"2020-12-18 11:28:26.000000000","message":"Ack","commit_id":"bdeb1753e093eb78195f04d7e0e027665d363a21"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":true,"context_lines":[{"line_number":2288,"context_line":"        self.assertTrue(context.cleaving_done)"},{"line_number":2289,"context_line":"        self.assertEqual(shard_ranges[4].upper_str, context.cursor)"},{"line_number":2290,"context_line":"        self.assertEqual(5, context.ranges_done)"},{"line_number":2291,"context_line":"        self.assertEqual(0, context.ranges_todo)"},{"line_number":2292,"context_line":""},{"line_number":2293,"context_line":"    def _check_complete_sharding(self, account, container, shard_bounds):"},{"line_number":2294,"context_line":"        broker \u003d self._make_sharding_broker("}],"source_content_type":"text/x-python","patch_set":14,"id":"2faeca8e_736ed81a","line":2291,"updated":"2020-12-18 16:36:41.000000000","message":"wow, thanks!","commit_id":"d277960161119face2eb9d617822b2645f19b2c1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad1801134eebb8f3c2b984778f4808110ce891c9","unresolved":true,"context_lines":[{"line_number":4434,"context_line":"        for state in (ShardRange.SHRUNK, ShardRange.SHARDED):"},{"line_number":4435,"context_line":"            shard_ranges \u003d self._make_shard_ranges(shard_bounds, state)"},{"line_number":4436,"context_line":"            for sr in shard_ranges:"},{"line_number":4437,"context_line":"                sr.set_deleted(Timestamp.now())"},{"line_number":4438,"context_line":"            broker.merge_shard_ranges(shard_ranges)"},{"line_number":4439,"context_line":"            with self._mock_sharder() as sharder:"},{"line_number":4440,"context_line":"                with mock.patch.object("}],"source_content_type":"text/x-python","patch_set":14,"id":"5c225518_44cb960c","line":4437,"updated":"2020-12-18 16:36:41.000000000","message":"oh yeah!!!","commit_id":"d277960161119face2eb9d617822b2645f19b2c1"}]}