)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"afca9439ff75718a027b35f7c66a24afe738dac4","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"  - Searches through to the shards for the latest set of ranges that"},{"line_number":14,"context_line":"    covers the namespace."},{"line_number":15,"context_line":"  - Goes inserts those into all the bad shards, those that represent"},{"line_number":16,"context_line":"    incorrect shards, which the state set to found. Also updates the bad"},{"line_number":17,"context_line":"    shard\u0027s own shard to trigger sharding."},{"line_number":18,"context_line":"  - Updates the root shard table to delete all the bad shards."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1f493fa4_6762ab40","line":15,"range":{"start_line":15,"start_character":9,"end_line":15,"end_character":46},"updated":"2020-04-24 04:09:43.000000000","message":"Oh, interesting -- so it\u0027ll actually make the network requests to get everyone squared?\n\nFWIW, the route I was going to investigate was to try updating *just the root DB*, then see if the shards would get themselves squared when they fetch their own range from the root. I *think* that could work, though the sharder may need a bit of updating.","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e38fa65a00ed47ebe03813a70b9c3aaedd47615","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"  - Searches through to the shards for the latest set of ranges that"},{"line_number":14,"context_line":"    covers the namespace."},{"line_number":15,"context_line":"  - Goes inserts those into all the bad shards, those that represent"},{"line_number":16,"context_line":"    incorrect shards, which the state set to found. Also updates the bad"},{"line_number":17,"context_line":"    shard\u0027s own shard to trigger sharding."},{"line_number":18,"context_line":"  - Updates the root shard table to delete all the bad shards."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1f493fa4_671aab7d","line":15,"range":{"start_line":15,"start_character":9,"end_line":15,"end_character":46},"in_reply_to":"1f493fa4_6762ab40","updated":"2020-04-24 04:39:17.000000000","message":"Yeah, I just made a big hammer :P\nI\u0027m sure there is a bunch of clean ups. And taking the shardrange state into account is a good idea.","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"afca9439ff75718a027b35f7c66a24afe738dac4","unresolved":false,"context_lines":[{"line_number":25,"context_line":"a container, so inserting them all is just much eaiser to code and this"},{"line_number":26,"context_line":"is a POC/WIP."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Looking at the sharder, when we\u0027ve finished sharding a shard (non root)"},{"line_number":29,"context_line":"we mark the shards own_shard as deleted, but the container itself"},{"line_number":30,"context_line":"doesn\u0027t seem to be. So that could be a bug."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: I25bc872a1a9c8f9ea11b2ca1c973d131cdae70f5"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1f493fa4_4717afa4","line":30,"range":{"start_line":28,"start_character":0,"end_line":30,"end_character":43},"updated":"2020-04-24 04:09:43.000000000","message":"IIRC, not a bug. If we mark the DB deleted immediately, we\u0027ll lose all metadata, including the sysmeta that says it\u0027s a shard. So instead we wait a reclaim age (which also gives us a chance to accept some in-flight async pendings and move them as misplaced objects). After *one* reclaim age, the sharder can mark it deleted, then after *another one* the replicator unlinks it.\n\nAgain, IIRC ;-)","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e38fa65a00ed47ebe03813a70b9c3aaedd47615","unresolved":false,"context_lines":[{"line_number":25,"context_line":"a container, so inserting them all is just much eaiser to code and this"},{"line_number":26,"context_line":"is a POC/WIP."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Looking at the sharder, when we\u0027ve finished sharding a shard (non root)"},{"line_number":29,"context_line":"we mark the shards own_shard as deleted, but the container itself"},{"line_number":30,"context_line":"doesn\u0027t seem to be. So that could be a bug."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: I25bc872a1a9c8f9ea11b2ca1c973d131cdae70f5"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1f493fa4_629bb9c8","line":30,"range":{"start_line":28,"start_character":0,"end_line":30,"end_character":43},"in_reply_to":"1f493fa4_4717afa4","updated":"2020-04-24 04:39:17.000000000","message":"oh, yeah, that\u0027s smart. and yeah is see it now in _audit_shard_container. Somehow missed that, somehow kept glazing over that bit of code. Nice","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"}],"swift/cli/shard_cleanup.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":26,"context_line":"conn_timeout \u003d 0.5"},{"line_number":27,"context_line":"node_timeout \u003d 10"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"class ShardCleanupException(Exception):"},{"line_number":30,"context_line":"    pass"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"logger \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_a241410e","line":29,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":29,"context_line":"class ShardCleanupException(Exception):"},{"line_number":30,"context_line":"    pass"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"logger \u003d None"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"def _make_parser():"},{"line_number":35,"context_line":"    parser \u003d argparse.ArgumentParser(description\u003d\u0027Manage shard ranges\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_8244051d","line":32,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E305 expected 2 blank lines after class or function definition, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"logger \u003d None"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"def _make_parser():"},{"line_number":35,"context_line":"    parser \u003d argparse.ArgumentParser(description\u003d\u0027Manage shard ranges\u0027)"},{"line_number":36,"context_line":"    parser.add_argument(\u0027root_container_db\u0027, help\u003d\"Path to the root container\")"},{"line_number":37,"context_line":"    parser.add_argument(\u0027container_ring\u0027, help\u003d\"Path to the container ring\")"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_e24a2925","line":34,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":44,"context_line":"    for r in ranges:"},{"line_number":45,"context_line":"        lower.add(str(r.lower))"},{"line_number":46,"context_line":"        upper.add(str(r.upper))"},{"line_number":47,"context_line":"    l \u003d lower.copy()"},{"line_number":48,"context_line":"    lower.difference_update(upper)"},{"line_number":49,"context_line":"    upper.difference_update(l)"},{"line_number":50,"context_line":"    return not list(zip(upper, lower))"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_c2456d17","line":47,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E741 ambiguous variable name \u0027l\u0027","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":49,"context_line":"    upper.difference_update(l)"},{"line_number":50,"context_line":"    return not list(zip(upper, lower))"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"def get_latest_ranges(shards):"},{"line_number":53,"context_line":"    def shard_timestamp(x):"},{"line_number":54,"context_line":"        return x.timestamp"},{"line_number":55,"context_line":"    latest_timestamp \u003d max(shards, key\u003dshard_timestamp)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_225531c7","line":52,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":55,"context_line":"    latest_timestamp \u003d max(shards, key\u003dshard_timestamp)"},{"line_number":56,"context_line":"    return {sr for sr in shards if sr.timestamp \u003d\u003d latest_timestamp.timestamp}"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"def get_latest_complete_set(shards):"},{"line_number":59,"context_line":"    shard_set \u003d set(shards)"},{"line_number":60,"context_line":"    latest_set \u003d get_latest_ranges(shard_set)"},{"line_number":61,"context_line":"    while not check_complete_ranges(latest_set):"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_0250f5d3","line":58,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":65,"context_line":"        latest_set.update(next_set)"},{"line_number":66,"context_line":"    return latest_set, shard_set.difference(latest_set)"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"def put_container(node, part, account, container, headers, body):"},{"line_number":69,"context_line":"    try:"},{"line_number":70,"context_line":"        direct_put_container(node, part, account, container,"},{"line_number":71,"context_line":"                             conn_timeout\u003dconn_timeout,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_6260d9a6","line":68,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":75,"context_line":"        msg \u003d \u0027Failed to put shard ranges to %s:%s/%s: %s\u0027,\\"},{"line_number":76,"context_line":"              node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], err.http_status"},{"line_number":77,"context_line":"        logger.warning(msg)"},{"line_number":78,"context_line":"        print(msg, file\u003dsys.stderr)"},{"line_number":79,"context_line":"    except (Exception, Timeout) as err:"},{"line_number":80,"context_line":"        msg \u003d \u0027Failed to put shard ranges to %s:%s/%s: %s\u0027,\\"},{"line_number":81,"context_line":"              node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], err"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_42631db2","line":78,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E999 SyntaxError: invalid syntax","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":85,"context_line":"        return True"},{"line_number":86,"context_line":"    return False"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"def send_shard_ranges(bad_shard, shard_ranges, ring, headers\u003dNone):"},{"line_number":89,"context_line":"    # Let\u0027s set the bad_shard\u0027s own shard to sharding state."},{"line_number":90,"context_line":"    now \u003d Timestamp.now()"},{"line_number":91,"context_line":"    bad_shard.state \u003d ShardRange.SHARDING"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_a25ae1ef","line":88,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":114,"context_line":"    results \u003d pool.waitall(None)"},{"line_number":115,"context_line":"    return results.count(True) \u003e\u003d quorum_size(ring.replica_count)"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"def main(args\u003dNone):"},{"line_number":118,"context_line":"    parser \u003d _make_parser()"},{"line_number":119,"context_line":"    args \u003d parser.parse_args(args)"},{"line_number":120,"context_line":"    logger \u003d get_logger({}, name\u003d\u0027ContainerBroker\u0027, log_to_console\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_825d25eb","line":117,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":124,"context_line":"    try:"},{"line_number":125,"context_line":"        broker.get_info()"},{"line_number":126,"context_line":"    except Exception as exc:"},{"line_number":127,"context_line":"        print(\u0027Error opening root container DB %s: %s\u0027 % (args.root_container_db, exc),"},{"line_number":128,"context_line":"              file\u003dsys.stderr)"},{"line_number":129,"context_line":"        return 2"},{"line_number":130,"context_line":"    print(\u0027Loaded db broker for %s.\u0027 % broker.path, file\u003dsys.stderr)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_e273497f","line":127,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E501 line too long (87 \u003e 79 characters)","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":148,"context_line":"          \"container replicator to push these deletes out.\")"},{"line_number":149,"context_line":"    broker.merge_shard_ranges(list(left))"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"},{"line_number":152,"context_line":"    exit(main())"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_c26e0d95","line":151,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: E305 expected 2 blank lines after class or function definition, found 1","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"7e83ad9313a6617e49e254bf55705a09b15373b1","unresolved":false,"context_lines":[{"line_number":149,"context_line":"    broker.merge_shard_ranges(list(left))"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"},{"line_number":152,"context_line":"    exit(main())"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f493fa4_226ed194","line":152,"updated":"2020-04-24 04:44:11.000000000","message":"pep8: W292 no newline at end of file","commit_id":"d961a6922d1450485db2b0222839205fe0f63e34"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"import argparse"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"class ShardCleanupException(Exception):"},{"line_number":21,"context_line":"    pass"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"logger \u003d None"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_7d2b582f","line":20,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":20,"context_line":"class ShardCleanupException(Exception):"},{"line_number":21,"context_line":"    pass"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"logger \u003d None"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"def _make_parser():"},{"line_number":26,"context_line":"    parser \u003d argparse.ArgumentParser(description\u003d\u0027Clean up shard ranges\u0027)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_dd21cc4f","line":23,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E305 expected 2 blank lines after class or function definition, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"logger \u003d None"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"def _make_parser():"},{"line_number":26,"context_line":"    parser \u003d argparse.ArgumentParser(description\u003d\u0027Clean up shard ranges\u0027)"},{"line_number":27,"context_line":"    parser.add_argument(\u0027root_container_db\u0027, help\u003d\"Path to the root container\")"},{"line_number":28,"context_line":"    return parser"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_bd1cc013","line":25,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":34,"context_line":"    for r in ranges:"},{"line_number":35,"context_line":"        lower.add(str(r.lower))"},{"line_number":36,"context_line":"        upper.add(str(r.upper))"},{"line_number":37,"context_line":"    l \u003d lower.copy()"},{"line_number":38,"context_line":"    lower.difference_update(upper)"},{"line_number":39,"context_line":"    upper.difference_update(l)"},{"line_number":40,"context_line":"    return not list(zip(upper, lower))"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_1d1cd410","line":37,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E741 ambiguous variable name \u0027l\u0027","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":39,"context_line":"    upper.difference_update(l)"},{"line_number":40,"context_line":"    return not list(zip(upper, lower))"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"def get_latest_ranges(shards):"},{"line_number":43,"context_line":"    def shard_timestamp(x):"},{"line_number":44,"context_line":"        return x.timestamp"},{"line_number":45,"context_line":"    latest_timestamp \u003d max(shards, key\u003dshard_timestamp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_fd1648f1","line":42,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":45,"context_line":"    latest_timestamp \u003d max(shards, key\u003dshard_timestamp)"},{"line_number":46,"context_line":"    return {sr for sr in shards if sr.timestamp \u003d\u003d latest_timestamp.timestamp}"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"def get_latest_complete_set(shards):"},{"line_number":49,"context_line":"    shard_set \u003d set(shards)"},{"line_number":50,"context_line":"    latest_set \u003d get_latest_ranges(shard_set)"},{"line_number":51,"context_line":"    while not check_complete_ranges(latest_set):"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_5dd3fc38","line":48,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":55,"context_line":"        latest_set.update(next_set)"},{"line_number":56,"context_line":"    return latest_set, shard_set.difference(latest_set)"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"def main(args\u003dNone):"},{"line_number":59,"context_line":"    parser \u003d _make_parser()"},{"line_number":60,"context_line":"    args \u003d parser.parse_args(args)"},{"line_number":61,"context_line":"    logger \u003d get_logger({}, name\u003d\u0027ContainerBroker\u0027, log_to_console\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_3dd6f028","line":58,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":64,"context_line":"    try:"},{"line_number":65,"context_line":"        broker.get_info()"},{"line_number":66,"context_line":"    except Exception as exc:"},{"line_number":67,"context_line":"        print(\u0027Error opening root container DB %s: %s\u0027 % (args.root_container_db, exc),"},{"line_number":68,"context_line":"              file\u003dsys.stderr)"},{"line_number":69,"context_line":"        return 2"},{"line_number":70,"context_line":"    print(\u0027Loaded db broker for %s.\u0027 % broker.path, file\u003dsys.stderr)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_9ddd8444","line":67,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E501 line too long (87 \u003e 79 characters)","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":65,"context_line":"        broker.get_info()"},{"line_number":66,"context_line":"    except Exception as exc:"},{"line_number":67,"context_line":"        print(\u0027Error opening root container DB %s: %s\u0027 % (args.root_container_db, exc),"},{"line_number":68,"context_line":"              file\u003dsys.stderr)"},{"line_number":69,"context_line":"        return 2"},{"line_number":70,"context_line":"    print(\u0027Loaded db broker for %s.\u0027 % broker.path, file\u003dsys.stderr)"},{"line_number":71,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_7de078fb","line":68,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E999 SyntaxError: invalid syntax","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06d5b0c069443fa52723a6b96b1d2b34ba122f5f","unresolved":false,"context_lines":[{"line_number":73,"context_line":"    if not broker.is_root_container():"},{"line_number":74,"context_line":"        print(\u0027Loaded db broker %s is not a root container.\u0027 % broker.path,"},{"line_number":75,"context_line":"              file\u003dsys.stderr)"},{"line_number":76,"context_line":"        sys.exit(1)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"    # step 1. Pull out the existing shards"},{"line_number":79,"context_line":"    shards \u003d [sr for sr in broker.get_shard_ranges() if not sr.deleted]"}],"source_content_type":"text/x-python","patch_set":2,"id":"bf51134e_8750db34","line":76,"updated":"2020-06-16 15:00:41.000000000","message":"So do we want to use return values, or calls to sys.exit?","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06d5b0c069443fa52723a6b96b1d2b34ba122f5f","unresolved":false,"context_lines":[{"line_number":83,"context_line":"        print(\u0027Error: {}\u0027.format(str(ex)), file\u003dsys.stderr)"},{"line_number":84,"context_line":"        sys.exit(2)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"    for bad_range in left:"},{"line_number":87,"context_line":"        # mark it as sharded so it\u0027ll be cleaned up"},{"line_number":88,"context_line":"        # (SHARDED and own_shard_range.deleted \u003d1 to be cleaned up in audit)"},{"line_number":89,"context_line":"        bad_range.state \u003d ShardRange.SHARDED"}],"source_content_type":"text/x-python","patch_set":2,"id":"bf51134e_2788efc4","line":86,"updated":"2020-06-16 15:00:41.000000000","message":"We should probably have a separate code path for\n\n if not left:","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"19648c75be989ec44ce76a39df7f30b4d05467c4","unresolved":false,"context_lines":[{"line_number":87,"context_line":"        # mark it as sharded so it\u0027ll be cleaned up"},{"line_number":88,"context_line":"        # (SHARDED and own_shard_range.deleted \u003d1 to be cleaned up in audit)"},{"line_number":89,"context_line":"        bad_range.state \u003d ShardRange.SHARDED"},{"line_number":90,"context_line":"        bad_range.state_timestamp \u003d Timestamp.now()"},{"line_number":91,"context_line":"        bad_range.set_deleted()"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    # now delete the bad ranges in this broker. We\u0027ll let replication push"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_829c055e","line":90,"updated":"2020-04-24 05:18:23.000000000","message":"This shardrange.SHARDED is done because it\u0027s what the sharder audit needs to be able to delete the shard, eventually. We could maybe just create a new state.","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06d5b0c069443fa52723a6b96b1d2b34ba122f5f","unresolved":false,"context_lines":[{"line_number":87,"context_line":"        # mark it as sharded so it\u0027ll be cleaned up"},{"line_number":88,"context_line":"        # (SHARDED and own_shard_range.deleted \u003d1 to be cleaned up in audit)"},{"line_number":89,"context_line":"        bad_range.state \u003d ShardRange.SHARDED"},{"line_number":90,"context_line":"        bad_range.state_timestamp \u003d Timestamp.now()"},{"line_number":91,"context_line":"        bad_range.set_deleted()"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    # now delete the bad ranges in this broker. We\u0027ll let replication push"}],"source_content_type":"text/x-python","patch_set":2,"id":"bf51134e_07cdeb5c","line":90,"in_reply_to":"1f493fa4_829c055e","updated":"2020-06-16 15:00:41.000000000","message":"Probably ought to print these out as we handle them. Maybe only w/ a --verbose flag?","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":96,"context_line":"          \"container replicator to push these deletes out.\")"},{"line_number":97,"context_line":"    broker.merge_shard_ranges(list(left))"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"},{"line_number":100,"context_line":"    exit(main())"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_dde6ec16","line":99,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: E305 expected 2 blank lines after class or function definition, found 1","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f95b94a6f125e0a5584bb53d5b69393fd4e90412","unresolved":false,"context_lines":[{"line_number":97,"context_line":"    broker.merge_shard_ranges(list(left))"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"},{"line_number":100,"context_line":"    exit(main())"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_bde160ff","line":100,"updated":"2020-04-24 06:33:21.000000000","message":"pep8: W292 no newline at end of file","commit_id":"5743e64855418cad97c248b111da822314a4aca3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6e4eb3f877be4340e1eb9859c59ec76297ff8d8a","unresolved":false,"context_lines":[{"line_number":39,"context_line":"        lower.add(str(r.lower))"},{"line_number":40,"context_line":"        upper.add(str(r.upper))"},{"line_number":41,"context_line":"    l \u003d lower.copy()"},{"line_number":42,"context_line":"    lower.difference_update(upper)"},{"line_number":43,"context_line":"    upper.difference_update(l)"},{"line_number":44,"context_line":"    return not list(zip(upper, lower))"},{"line_number":45,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_231ddfbd","line":42,"updated":"2020-09-09 19:45:57.000000000","message":"this is like `lower \u003d lower - upper` right?","commit_id":"69341eb6c5b74a5127e9796924565061623db790"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6e4eb3f877be4340e1eb9859c59ec76297ff8d8a","unresolved":false,"context_lines":[{"line_number":41,"context_line":"    l \u003d lower.copy()"},{"line_number":42,"context_line":"    lower.difference_update(upper)"},{"line_number":43,"context_line":"    upper.difference_update(l)"},{"line_number":44,"context_line":"    return not list(zip(upper, lower))"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"def get_latest_ranges(shards):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f560f44_c3686322","line":44,"updated":"2020-09-09 19:45:57.000000000","message":"I don\u0027t understand how this validates the shard ranges are a complete run MIN to MAX?  The passed in ranges could have overlaps or at least duplicates, right?","commit_id":"69341eb6c5b74a5127e9796924565061623db790"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"19648c75be989ec44ce76a39df7f30b4d05467c4","unresolved":false,"context_lines":[{"line_number":1024,"context_line":"    def _make_misplaced_object_bounds(self, broker):"},{"line_number":1025,"context_line":"        bounds \u003d []"},{"line_number":1026,"context_line":"        state \u003d broker.get_db_state()"},{"line_number":1027,"context_line":"        if state \u003d\u003d SHARDED or broker.get_own_shard_range().deleted:"},{"line_number":1028,"context_line":"            # Anything in the object table is treated as a misplaced object."},{"line_number":1029,"context_line":"            bounds.append((\u0027\u0027, \u0027\u0027))"},{"line_number":1030,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"1f493fa4_c29d6d58","line":1027,"updated":"2020-04-24 05:18:23.000000000","message":"This SHARDED is the db_state, not to be confused with the shardrange state :/\n\nAdd thing so we can get the _all_ bounds to clean up the bad shards.","commit_id":"5743e64855418cad97c248b111da822314a4aca3"}]}