)]}'
{"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05d61c2d960a171f866bf3b1f9c97410bfed1f9","unresolved":true,"context_lines":[{"line_number":477,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":478,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":479,"context_line":"    incomplete_paths \u003d [path for path in ranked_paths"},{"line_number":480,"context_line":"                        if not path.includes(own_sr)]"},{"line_number":481,"context_line":""},{"line_number":482,"context_line":"    if not complete_paths:"},{"line_number":483,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"}],"source_content_type":"text/x-python","patch_set":10,"id":"2dcc27b9_7fc500db","line":480,"updated":"2021-01-27 06:31:43.000000000","message":"Oh cool, that\u0027s a different way of checking for complete and imcomplete paths. That makes me feel better about how we\u0027re dealing with gaps.","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03d820c5dfb2138031cbee077c0091b71d994ee8","unresolved":true,"context_lines":[{"line_number":477,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":478,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":479,"context_line":"    incomplete_paths \u003d [path for path in ranked_paths"},{"line_number":480,"context_line":"                        if not path.includes(own_sr)]"},{"line_number":481,"context_line":""},{"line_number":482,"context_line":"    if not complete_paths:"},{"line_number":483,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"}],"source_content_type":"text/x-python","patch_set":10,"id":"229beac0_acaf2396","line":480,"in_reply_to":"2dcc27b9_7fc500db","updated":"2021-01-27 12:33:10.000000000","message":"yep, all paths are continuous, so complete means they span the entire namespace","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05d61c2d960a171f866bf3b1f9c97410bfed1f9","unresolved":true,"context_lines":[{"line_number":483,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"},{"line_number":484,"context_line":"        print(\u0027Repairs necessary to fill gaps.\u0027)"},{"line_number":485,"context_line":"        print(\u0027Gap filling not supported by this tool. No repairs performed.\u0027)"},{"line_number":486,"context_line":"        return 1, None"},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"    if len(ranked_paths) \u003d\u003d 1:"},{"line_number":489,"context_line":"        print(\u0027Found one complete sequence of %d shard ranges and no \u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"b38add4d_563ea166","line":486,"updated":"2021-01-27 06:31:43.000000000","message":"Cool, and we\u0027re dropping if there are. Sweet!","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03d820c5dfb2138031cbee077c0091b71d994ee8","unresolved":true,"context_lines":[{"line_number":483,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"},{"line_number":484,"context_line":"        print(\u0027Repairs necessary to fill gaps.\u0027)"},{"line_number":485,"context_line":"        print(\u0027Gap filling not supported by this tool. No repairs performed.\u0027)"},{"line_number":486,"context_line":"        return 1, None"},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"    if len(ranked_paths) \u003d\u003d 1:"},{"line_number":489,"context_line":"        print(\u0027Found one complete sequence of %d shard ranges and no \u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"cb7f0771_f99cc5e0","line":486,"in_reply_to":"b38add4d_563ea166","updated":"2021-01-27 12:33:10.000000000","message":"yes, I don\u0027t want to get embroiled in gap fixing while the known problems are overlaps of complete paths.","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":533,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":534,"context_line":"    # path that reached them."},{"line_number":535,"context_line":"    cleaved_states \u003d (ShardRange.CLEAVED, ShardRange.ACTIVE)"},{"line_number":536,"context_line":"    cleaved_upper \u003d acceptor_path.filter_states(cleaved_states).upper"},{"line_number":537,"context_line":"    beyond_cleaved \u003d acceptor_path.filter(marker\u003dcleaved_upper)"},{"line_number":538,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":539,"context_line":"        raise InvalidSolutionException("}],"source_content_type":"text/x-python","patch_set":12,"id":"7e0f4faa_1784c0ae","line":536,"updated":"2021-02-04 06:11:24.000000000","message":"OK here we\u0027re checking that the chosen path is actually cleaved/active completely, the filter_states will actaully break when it hits a SR that isn\u0027t in the filter.\n\nCool I get it, nice. I guess we need to be sure they\u0027re sorted, but that shold be the case because of find_paths.","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"26d9086fe588b760b636a2ba1d0f28f370d0c4dc","unresolved":true,"context_lines":[{"line_number":533,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":534,"context_line":"    # path that reached them."},{"line_number":535,"context_line":"    cleaved_states \u003d (ShardRange.CLEAVED, ShardRange.ACTIVE)"},{"line_number":536,"context_line":"    cleaved_upper \u003d acceptor_path.filter_states(cleaved_states).upper"},{"line_number":537,"context_line":"    beyond_cleaved \u003d acceptor_path.filter(marker\u003dcleaved_upper)"},{"line_number":538,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":539,"context_line":"        raise InvalidSolutionException("}],"source_content_type":"text/x-python","patch_set":12,"id":"ce10ea5d_dc6c1965","line":536,"in_reply_to":"7e0f4faa_1784c0ae","updated":"2021-02-04 19:13:58.000000000","message":"no, we\u0027re checking that there are no cleaved or active ranges further down the chosen path than the end of the first run of cleaved/active i.e.:\n\n(active, active, active) is OK\n(cleaved, cleaved, created) is OK\n(cleaved, created, cleaved) is not OK (weird scenario, need human help)","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":543,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":544,"context_line":"        raise InvalidSolutionException("},{"line_number":545,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":546,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":547,"context_line":""},{"line_number":548,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":549,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"c5783009_d8a822cb","line":546,"updated":"2021-02-04 06:11:24.000000000","message":"So if it turns out that the top ranked path is an invalid solution, rather then just end, sholdn\u0027t we check the next ranked path, as it might actaully also be complete and correct?\n\nOr are we just going to wait and hope that the cleaving/active states get better and run again later. I worry that there actually still is a good path but somehow our sorting picked something slightly broken.","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"26d9086fe588b760b636a2ba1d0f28f370d0c4dc","unresolved":true,"context_lines":[{"line_number":543,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":544,"context_line":"        raise InvalidSolutionException("},{"line_number":545,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":546,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":547,"context_line":""},{"line_number":548,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":549,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"97f9562a_ac8465b5","line":546,"in_reply_to":"c5783009_d8a822cb","updated":"2021-02-04 19:13:58.000000000","message":"these invalid solutions imply that the set of shard ranges is in a state that we do not anticipate, so for now I am inclined to bail out and let humans help.\n\nThe next best path will have cleaved to \u003c\u003d cleaved_upper so will fail for the same reasons as the chosen_path - there\u0027s a stray range in cleaved/active that is beyond cleaved_upper - we should probably investigate such a scenario and understand what happened.","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":false,"context_lines":[{"line_number":555,"context_line":"              (i, donor, donor.object_count))"},{"line_number":556,"context_line":"    print(\u0027Acceptors:\u0027)"},{"line_number":557,"context_line":"    for i, acceptor in enumerate(acceptor_path):"},{"line_number":558,"context_line":"        print(\u0027    [%d] %s\u0027 % (i, acceptor))"},{"line_number":559,"context_line":""},{"line_number":560,"context_line":""},{"line_number":561,"context_line":"def find_repair_solution(shard_ranges, own_sr, args):"}],"source_content_type":"text/x-python","patch_set":12,"id":"51112234_f074d6ff","line":558,"updated":"2021-02-04 06:11:24.000000000","message":"I look forward to playing with this in the future sometime to add the graphviz stuff, I think that would look awesome not just as a tool but as a display in a control interface somewhere 😊","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2cb5fc90dff830efceec5e1fd97e88ea78a7ba10","unresolved":true,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    paths \u003d find_paths(shard_ranges)"},{"line_number":529,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":530,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":531,"context_line":"    if not complete_paths:"},{"line_number":532,"context_line":"        raise GapsFoundException"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":20,"id":"7ed7a6ff_3be10bc2","line":530,"updated":"2021-03-17 05:50:53.000000000","message":"includes here will only find gaps and the start and end of the path right. Because it\u0027s only looking at the max and min.\n\nMaybe we need to start using find_missing_ranges which does just this, see: https://review.opendev.org/c/openstack/swift/+/781013\n\nWhich you\u0027re more the welcome to squash in.","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67813d1ac3343ffcafc4b68b8041e78da3f73cb8","unresolved":true,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    paths \u003d find_paths(shard_ranges)"},{"line_number":529,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":530,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":531,"context_line":"    if not complete_paths:"},{"line_number":532,"context_line":"        raise GapsFoundException"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":20,"id":"e455eed9_6432bbfb","line":530,"in_reply_to":"7ed7a6ff_3be10bc2","updated":"2021-03-17 18:31:15.000000000","message":"paths don\u0027t have gaps, other than at the start and end w.r.t. the entire namespace. This is a fundamental: a path is always continuous but not necessarily complete. Also, if there is no gap in shard_ranges then there must exist a path that spans the entire namespace.\n\nI\u0027m adding this comment:\n\n  # individual paths do not have gaps within them; if no path spans the\n  # entire namespace then there must be a gap in the shard_ranges","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2f4d29371957ae437b6d6eba75fa6809b998f077","unresolved":true,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    paths \u003d find_paths(shard_ranges)"},{"line_number":529,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":530,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":531,"context_line":"    if not complete_paths:"},{"line_number":532,"context_line":"        raise GapsFoundException"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":20,"id":"bf088346_4b0af113","line":530,"in_reply_to":"e455eed9_6432bbfb","updated":"2021-03-18 04:14:58.000000000","message":"Oh right, of course! As a path with a gap in the middle would then appear are 2 paths.. which isn\u0027t incorrect. That makes alot of sense!","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2cb5fc90dff830efceec5e1fd97e88ea78a7ba10","unresolved":true,"context_lines":[{"line_number":575,"context_line":"    except GapsFoundException:"},{"line_number":576,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"},{"line_number":577,"context_line":"        print(\u0027Repairs necessary to fill gaps.\u0027)"},{"line_number":578,"context_line":"        print(\u0027Gap filling not supported by this tool. No repairs performed.\u0027)"},{"line_number":579,"context_line":"        raise"},{"line_number":580,"context_line":"    except InvalidStateException as exc:"},{"line_number":581,"context_line":"        print(\u0027WARNING: %s\u0027 % exc)"}],"source_content_type":"text/x-python","patch_set":20,"id":"34ff6519_20bbe17b","line":578,"updated":"2021-03-17 05:50:53.000000000","message":"Might be out of scope, but I wonder if we should at least print the gaps which could be passed back in the GapFoundException, because if we ever find outself in this situtaiton it would be what we\u0027d need to know next for any investigations.\n\nThough I guess this could be something we should see pretty easily through graph visualisation.","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67813d1ac3343ffcafc4b68b8041e78da3f73cb8","unresolved":true,"context_lines":[{"line_number":575,"context_line":"    except GapsFoundException:"},{"line_number":576,"context_line":"        print(\u0027Found no complete sequence of shard ranges.\u0027)"},{"line_number":577,"context_line":"        print(\u0027Repairs necessary to fill gaps.\u0027)"},{"line_number":578,"context_line":"        print(\u0027Gap filling not supported by this tool. No repairs performed.\u0027)"},{"line_number":579,"context_line":"        raise"},{"line_number":580,"context_line":"    except InvalidStateException as exc:"},{"line_number":581,"context_line":"        print(\u0027WARNING: %s\u0027 % exc)"}],"source_content_type":"text/x-python","patch_set":20,"id":"227ec8b6_9b906aec","line":578,"in_reply_to":"34ff6519_20bbe17b","updated":"2021-03-17 18:31:15.000000000","message":"totally agree that would be the next thing we\u0027d need, but I\u0027d prefer to leave that for a follow on patch, since this patch is already large and has been hanging around for a long time 😊","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2cb5fc90dff830efceec5e1fd97e88ea78a7ba10","unresolved":true,"context_lines":[{"line_number":650,"context_line":""},{"line_number":651,"context_line":""},{"line_number":652,"context_line":"def analyze_shard_ranges(args):"},{"line_number":653,"context_line":"    shard_data \u003d _load_and_validate_shard_data(args, require_index\u003dFalse)"},{"line_number":654,"context_line":"    shard_ranges \u003d [ShardRange.from_dict(data) for data in shard_data]"},{"line_number":655,"context_line":"    whole_sr \u003d ShardRange(\u0027whole/namespace\u0027, 0)"},{"line_number":656,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":20,"id":"0b8495a2_99ada8e0","line":653,"updated":"2021-03-17 05:50:53.000000000","message":"Why must we use a json to provide the shard data? for are we expected to do a \u0027show | analyze\u0027.","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67813d1ac3343ffcafc4b68b8041e78da3f73cb8","unresolved":true,"context_lines":[{"line_number":650,"context_line":""},{"line_number":651,"context_line":""},{"line_number":652,"context_line":"def analyze_shard_ranges(args):"},{"line_number":653,"context_line":"    shard_data \u003d _load_and_validate_shard_data(args, require_index\u003dFalse)"},{"line_number":654,"context_line":"    shard_ranges \u003d [ShardRange.from_dict(data) for data in shard_data]"},{"line_number":655,"context_line":"    whole_sr \u003d ShardRange(\u0027whole/namespace\u0027, 0)"},{"line_number":656,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":20,"id":"502b6b77_486cb75e","line":653,"in_reply_to":"0b8495a2_99ada8e0","updated":"2021-03-17 18:31:15.000000000","message":"repair without a \u0027yes\u0027 does the same on a db as analyze does on json, so analyze is only here to cater for loading from json rather than a db.\n\nI gave it the broad name \u0027analyze\u0027 because I can imagine it reporting other interesting properties in the future (like gaps, \u0027raggedness\u0027 etc??)\n\nSo, yes it is expected that it would be used with s-m-s-r show | s-m-s-r analyze.\n\nIt would be cool if analyze could accept either a db or a json file, particularly if it grows other features, but it\u0027s already beyond the immediate use case of repair, so maybe for follow-on. For now repeat without a yes gives us everything that analyze might for a db.\n\noff-topic: I do think that a --dry-run option would be useful for repair and compact, to supress the \u0027yes?\u0027 prompt.","commit_id":"6b177ef047877ad88fb6c51b4eb8404e4517c335"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":534,"context_line":"        raise GapsFoundException"},{"line_number":535,"context_line":""},{"line_number":536,"context_line":"    # simple repair strategy: choose the highest ranked complete sequence and"},{"line_number":537,"context_line":"    # shrink all other shard ranges into it"},{"line_number":538,"context_line":"    acceptor_path \u003d ranked_paths[0]"},{"line_number":539,"context_line":"    acceptor_names \u003d set(sr.name for sr in acceptor_path)"},{"line_number":540,"context_line":"    overlapping_donors \u003d ShardRangeList([sr for sr in shard_ranges"}],"source_content_type":"text/x-python","patch_set":21,"id":"957066c6_9e74fb5d","line":537,"updated":"2021-03-18 04:57:54.000000000","message":"+1 always best to start with the simple approach. we can always optimise later.","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":546,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":547,"context_line":"    # path that reached them."},{"line_number":548,"context_line":"    cleaved_states \u003d (ShardRange.CLEAVED, ShardRange.ACTIVE)"},{"line_number":549,"context_line":"    cleaved_upper \u003d acceptor_path.filter_states(cleaved_states).upper"},{"line_number":550,"context_line":"    beyond_cleaved \u003d acceptor_path.filter(marker\u003dcleaved_upper)"},{"line_number":551,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":552,"context_line":"        raise InvalidSolutionException("}],"source_content_type":"text/x-python","patch_set":21,"id":"2f896494_4e6f02b3","line":549,"updated":"2021-03-18 04:57:54.000000000","message":"Love what we\u0027re testing here, but not 100% sure about the name filter_state, as that in my mind implies eactly what the filter on the line below does. Ie, filter the path for all the ranges in the given state. But this isn\u0027t what filter_states does. Well not strictly, it does if you pass in \u0027allow_gaps\u0027. Otherwise it walks the path of shardranges and stop when it first gets to a shardrange not in the states.\n\nI just think the name doesn\u0027t suit. Maybe `walk_filter(states)` or reverse it and make the param tell us more on what we\u0027re doing: \n\n  cleaved_upper \u003d acceptor_path.filter_states(states, walk_until\u003dTrue)\n\nAnd by default filter with gaps, as Id assume if you\u0027d used a list and saids filter by state you\u0027d expect all to have a list of ranges with that state (with gaps).","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"11ff60d3d9217a1bd451956d051b8aac24ac02ae","unresolved":true,"context_lines":[{"line_number":546,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":547,"context_line":"    # path that reached them."},{"line_number":548,"context_line":"    cleaved_states \u003d (ShardRange.CLEAVED, ShardRange.ACTIVE)"},{"line_number":549,"context_line":"    cleaved_upper \u003d acceptor_path.filter_states(cleaved_states).upper"},{"line_number":550,"context_line":"    beyond_cleaved \u003d acceptor_path.filter(marker\u003dcleaved_upper)"},{"line_number":551,"context_line":"    if beyond_cleaved.filter_states(cleaved_states):"},{"line_number":552,"context_line":"        raise InvalidSolutionException("}],"source_content_type":"text/x-python","patch_set":21,"id":"dad5e491_401dc74d","line":549,"in_reply_to":"2f896494_4e6f02b3","updated":"2021-03-18 20:07:21.000000000","message":"yep, I\u0027ve been fretting about the name filter_states too. certainly agree that flipping the default might make sense. It\u0027s a such a handy method to have, but not easy to name!\n\nupdate: rewrote it as something much simpler in next revision","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":558,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":559,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":560,"context_line":""},{"line_number":561,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":562,"context_line":""},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"def print_repair_solution(acceptor_path, overlapping_donors):"}],"source_content_type":"text/x-python","patch_set":21,"id":"a03064d0_ad3aa815","line":561,"updated":"2021-03-18 04:57:54.000000000","message":"I wonder if this could be moved somehow into sharder, as I\u0027de love to use this for the graph visualiser, also it would be nice to have the same check/analyze/repair/fsck code for both the tool and the auto-sharder (once we integrate it).","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"11ff60d3d9217a1bd451956d051b8aac24ac02ae","unresolved":true,"context_lines":[{"line_number":558,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":559,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":560,"context_line":""},{"line_number":561,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":562,"context_line":""},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"def print_repair_solution(acceptor_path, overlapping_donors):"}],"source_content_type":"text/x-python","patch_set":21,"id":"8e31afb8_a0de2ac4","line":561,"in_reply_to":"a03064d0_ad3aa815","updated":"2021-03-18 20:07:21.000000000","message":"that should be possible","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":698,"context_line":"def _add_yes_arg(parser):"},{"line_number":699,"context_line":"    parser.add_argument("},{"line_number":700,"context_line":"        \u0027--yes\u0027, \u0027-y\u0027, action\u003d\u0027store_true\u0027, default\u003dFalse,"},{"line_number":701,"context_line":"        help\u003d\u0027Apply shard range changes to broker without prompting.\u0027)"},{"line_number":702,"context_line":""},{"line_number":703,"context_line":""},{"line_number":704,"context_line":"def _make_parser():"}],"source_content_type":"text/x-python","patch_set":21,"id":"3c9d2f32_36a1414c","line":701,"updated":"2021-03-18 04:57:54.000000000","message":"good idea, rather then adding it here and there.","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":892,"context_line":"        args.rows_per_shard \u003d rows_per_shard"},{"line_number":893,"context_line":""},{"line_number":894,"context_line":"    if args.func in (analyze_shard_ranges,):"},{"line_number":895,"context_line":"        args.input \u003d args.path_to_file"},{"line_number":896,"context_line":"        return args.func(args) or 0"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":"    logger \u003d get_logger({}, name\u003d\u0027ContainerBroker\u0027, log_to_console\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":21,"id":"79c3c836_d758a7f1","line":895,"updated":"2021-03-18 04:57:54.000000000","message":"Did you just want to change repair to use path_to_file too, rather then doing this here? esp now that\u0027s it has a multifunctional name.","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"11ff60d3d9217a1bd451956d051b8aac24ac02ae","unresolved":true,"context_lines":[{"line_number":892,"context_line":"        args.rows_per_shard \u003d rows_per_shard"},{"line_number":893,"context_line":""},{"line_number":894,"context_line":"    if args.func in (analyze_shard_ranges,):"},{"line_number":895,"context_line":"        args.input \u003d args.path_to_file"},{"line_number":896,"context_line":"        return args.func(args) or 0"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":"    logger \u003d get_logger({}, name\u003d\u0027ContainerBroker\u0027, log_to_console\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":21,"id":"8a05cf25_6ae7ce6b","line":895,"in_reply_to":"79c3c836_d758a7f1","updated":"2021-03-18 20:07:21.000000000","message":"repair does use path_to_file, the remapping here is because the functions that load data from file expect args.input, which for some commands (replace?) is different to args.path_to_file, i.e. you\u0027d write:\n\n\n  s-m-s-r db_file replace json-file\n  \u003cpath_to_file\u003e \u003cinput\u003e \u003ccommand\u003e\n\nwhereas you\u0027d write:\n\n  s-m-s-r json_file analyze\n\nso I needed to map input-\u003epath_to_file","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"d9c913235defcbfde8015b0e632756ca7485c2fc","unresolved":true,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    paths \u003d find_paths(shard_ranges)"},{"line_number":529,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":530,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":531,"context_line":"    if not complete_paths:"},{"line_number":532,"context_line":"        # individual paths do not have gaps within them; if no path spans the"},{"line_number":533,"context_line":"        # entire namespace then there must be a gap in the shard_ranges"}],"source_content_type":"text/x-python","patch_set":23,"id":"241e32cf_03c98f5a","line":530,"updated":"2021-03-24 02:17:23.000000000","message":"Not like it matters, but you don\u0027t have to comprehend the whole list, I think. If ranked_paths[0].includes(own_sr) then you have at least one. You\u0027re actually using this property of ranking just 3 lines below in acceptor_path \u003d ranked_paths[0].","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"25f493c00529fa04d42fba234f97610515595f88","unresolved":false,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    paths \u003d find_paths(shard_ranges)"},{"line_number":529,"context_line":"    ranked_paths \u003d rank_paths(paths, own_sr)"},{"line_number":530,"context_line":"    complete_paths \u003d [path for path in ranked_paths if path.includes(own_sr)]"},{"line_number":531,"context_line":"    if not complete_paths:"},{"line_number":532,"context_line":"        # individual paths do not have gaps within them; if no path spans the"},{"line_number":533,"context_line":"        # entire namespace then there must be a gap in the shard_ranges"}],"source_content_type":"text/x-python","patch_set":23,"id":"29f3c1b7_b6182009","line":530,"in_reply_to":"241e32cf_03c98f5a","updated":"2021-03-25 12:54:07.000000000","message":"Done","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"da65a3de3acd05cae5918bf86b655838abb3cf13","unresolved":true,"context_lines":[{"line_number":544,"context_line":"    # progressed continuous cleaving, which has reached cleaved_upper, then we"},{"line_number":545,"context_line":"    # don\u0027t expect any shard ranges beyond cleaved_upper to be in states"},{"line_number":546,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":547,"context_line":"    # path that reached them."},{"line_number":548,"context_line":"    cleaved_states \u003d {ShardRange.CLEAVED, ShardRange.ACTIVE}"},{"line_number":549,"context_line":"    cleaved_upper \u003d acceptor_path.find_lower("},{"line_number":550,"context_line":"        lambda sr: sr.state not in cleaved_states)"}],"source_content_type":"text/x-python","patch_set":23,"id":"8a14932d_935f7281","line":547,"updated":"2021-03-19 05:53:35.000000000","message":"Just thinking out loud, could we have 2 sharding shards? This could make it look like 2 cleaving has happened later in the path?\n\nOh except we are blocking it entirely if there is a SHARDING range. So what are we expecting here? FOUND, CREATED I guess, but don\u0027t both of those indicate something should be SHARDING?\n\nIn fact, doesn\u0027t something in the CLEAVED state means something \"could\" stil be sharding. Although CLEAVED does means it\u0027s happily in charge of it\u0027s namespace.\n\nI guess it just means I don\u0027t expect this code to trigger much so long as we have the SHARDING bloc active at the start of the method.","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a60ef8d4483109ca3e7fe8f54696087dd2bc7732","unresolved":true,"context_lines":[{"line_number":544,"context_line":"    # progressed continuous cleaving, which has reached cleaved_upper, then we"},{"line_number":545,"context_line":"    # don\u0027t expect any shard ranges beyond cleaved_upper to be in states"},{"line_number":546,"context_line":"    # CLEAVED or ACTIVE, otherwise there should have been a better acceptor"},{"line_number":547,"context_line":"    # path that reached them."},{"line_number":548,"context_line":"    cleaved_states \u003d {ShardRange.CLEAVED, ShardRange.ACTIVE}"},{"line_number":549,"context_line":"    cleaved_upper \u003d acceptor_path.find_lower("},{"line_number":550,"context_line":"        lambda sr: sr.state not in cleaved_states)"}],"source_content_type":"text/x-python","patch_set":23,"id":"e8566ffd_4f2eed51","line":547,"in_reply_to":"8a14932d_935f7281","updated":"2021-03-22 18:50:47.000000000","message":"The SHARDING guard does not cover own shard range. We *do* expect this code to execute when a container has stalled part way through because of overlaps. Roots (at least) fail audit if there are overlaps and therefore may not progress with cleaving. I\u0027m pretty sure we\u0027ve seen this in production.\n\nBut I am being defensive about not attempting repair if we know there are sharding *shards*.","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fb450c6c1a74f50d52718c03a3a514c88bb97ed7","unresolved":true,"context_lines":[{"line_number":557,"context_line":"    if beyond_cleaved.states.intersection(cleaved_states):"},{"line_number":558,"context_line":"        raise InvalidSolutionException("},{"line_number":559,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":560,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":563,"context_line":""}],"source_content_type":"text/x-python","patch_set":25,"id":"6c8e1d9c_32cccb26","line":560,"updated":"2021-03-29 04:47:34.000000000","message":"Id expect donor ranges to have ACTIVE or cleaved states in it. But I guess we shouldn\u0027t get one beyond the accptor path. Unless maybe we\u0027ve had a split-brain and a smaller side shards. When we come back together could it pick the bigger one (that maybe hasn\u0027t finished). but in that case what is the right solution? Picking the bigger not quite sharded path or the smaller.\n\nI guess the set of ranges wouldn\u0027t even be accepted until sharding is comepleted so probably a moot point.","commit_id":"c9c42c07c9c6d354bd60b2e2d64b3db628b8e8f3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4a07c6db4664e5e9954bf87fb6a7f3a3e5e8b2ee","unresolved":true,"context_lines":[{"line_number":557,"context_line":"    if beyond_cleaved.states.intersection(cleaved_states):"},{"line_number":558,"context_line":"        raise InvalidSolutionException("},{"line_number":559,"context_line":"            \u0027Isolated cleaved and/or active shard ranges in donor ranges\u0027,"},{"line_number":560,"context_line":"            acceptor_path, overlapping_donors)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    return acceptor_path, overlapping_donors"},{"line_number":563,"context_line":""}],"source_content_type":"text/x-python","patch_set":25,"id":"a6caa67d_06e0aceb","line":560,"in_reply_to":"6c8e1d9c_32cccb26","updated":"2021-03-29 10:03:26.000000000","message":"Ranking prefers complete paths and then cleaving_progress, so we\u0027d choose the \u0027smaller\u0027 side (assuming you mean object count) if it has completed cleaving over the bigger one if it hasn\u0027t.\n\nIIRC my thinking re. cleaving progress was to avoid choosing a path that weaved its way from cleaved to uncleaved and back again, because that presents the problem of making isolated inactive shard ranges retrospectively active (retrospectively w.r.t. the cleaving context cursor).  I wanted the good path to be representative of an \u0027expected\u0027 shard range set, so not have CLEAVED-CREATED-CLEAVED for example. It then made sense to choose the path that was most cleaved, to avoid the solution shrinking a CLEAVED range into a yet-to-be-cleaved range and figuring out what should then happen to the acceptor state.\n\nTo put it another way, I am trying to make it that acceptor states are at least as advanced as their donors, as far as CLEAVED and ACTIVE are concerned.\n\nBut, we\u0027re stumbling across the \u0027small replica shards (too) quickly\u0027 issue in several places (e.g. here, shrinking decisions), so more motivation to find a fix for that.","commit_id":"c9c42c07c9c6d354bd60b2e2d64b3db628b8e8f3"}],"swift/common/utils.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":5539,"context_line":"        return set(sr.timestamp for sr in self)"},{"line_number":5540,"context_line":""},{"line_number":5541,"context_line":"    def get_bounds(self):"},{"line_number":5542,"context_line":"        return [sr.lower for sr in self] + [self.upper]"},{"line_number":5543,"context_line":""},{"line_number":5544,"context_line":"    def includes(self, other):"},{"line_number":5545,"context_line":"        return self.lower \u003c\u003d other.lower and self.upper \u003e\u003d other.upper"}],"source_content_type":"text/x-python","patch_set":12,"id":"180a8c0c_86f476de","line":5542,"updated":"2021-02-04 06:11:24.000000000","message":"If there is a gap in the shard range list, could we end up with a missing upper bound in this list of get_bounds?","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"26d9086fe588b760b636a2ba1d0f28f370d0c4dc","unresolved":true,"context_lines":[{"line_number":5539,"context_line":"        return set(sr.timestamp for sr in self)"},{"line_number":5540,"context_line":""},{"line_number":5541,"context_line":"    def get_bounds(self):"},{"line_number":5542,"context_line":"        return [sr.lower for sr in self] + [self.upper]"},{"line_number":5543,"context_line":""},{"line_number":5544,"context_line":"    def includes(self, other):"},{"line_number":5545,"context_line":"        return self.lower \u003c\u003d other.lower and self.upper \u003e\u003d other.upper"}],"source_content_type":"text/x-python","patch_set":12,"id":"a017dc89_07999fc1","line":5542,"in_reply_to":"180a8c0c_86f476de","updated":"2021-02-04 19:13:58.000000000","message":"turns out I don\u0027t use this - think I put it in while debugging - so I will delete","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"d9c913235defcbfde8015b0e632756ca7485c2fc","unresolved":true,"context_lines":[{"line_number":5026,"context_line":"    @functools.total_ordering"},{"line_number":5027,"context_line":"    class MaxBound(OuterBound):"},{"line_number":5028,"context_line":"        def __hash__(self):"},{"line_number":5029,"context_line":"            return 1"},{"line_number":5030,"context_line":""},{"line_number":5031,"context_line":"        def __ge__(self, other):"},{"line_number":5032,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":23,"id":"50d0b228_549eaac9","line":5029,"updated":"2021-03-24 02:17:23.000000000","message":"Very curious. Will ask offline.","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"}],"swift/container/backend.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"d9c913235defcbfde8015b0e632756ca7485c2fc","unresolved":true,"context_lines":[{"line_number":1388,"context_line":"        \"\"\""},{"line_number":1389,"context_line":"        if not shard_ranges:"},{"line_number":1390,"context_line":"            return"},{"line_number":1391,"context_line":"        if not isinstance(shard_ranges, (list, ShardRangeList)):"},{"line_number":1392,"context_line":"            shard_ranges \u003d [shard_ranges]"},{"line_number":1393,"context_line":""},{"line_number":1394,"context_line":"        item_list \u003d []"}],"source_content_type":"text/x-python","patch_set":23,"id":"5f1f0d11_973e1b64","line":1391,"updated":"2021-03-24 02:17:23.000000000","message":"Okay, although once you have 3 of them, I\u0027ll ask for something like getattr(shard_ranges, \u0027__iter__\u0027, None) perhaps.","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"072d0b9c02f723256cb617e876e33c57b078d80e","unresolved":true,"context_lines":[{"line_number":324,"context_line":"    paths_to_node \u003d collections.defaultdict(list)"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"    # visit the nodes in ascending order by name..."},{"line_number":327,"context_line":"    for node, edges in sorted(node_successors.items()):"},{"line_number":328,"context_line":"        if not edges:"},{"line_number":329,"context_line":"            # this node is a dead-end, so there\u0027s no path updates to make"},{"line_number":330,"context_line":"            continue"}],"source_content_type":"text/x-python","patch_set":7,"id":"a5e569b3_04a23093","line":327,"updated":"2021-01-10 23:51:33.000000000","message":"Seems you\u0027ve built it on the asusmption it can take any list of shards, and we can analyze it. Pretty cool, however, would we ever not want to run this on root containers. I mean if we are assuming root containers should have an idea of the state of all the shards. And shards themselves even if split brained while sharding, I\u0027d assume would just shard themselves the best they can and then delete.\n\nI guess I worry that if we somehow have a gap (missing first or last shard) we\u0027d do work but still silently have a gap. Where as with the root container we know we\u0027re dealing with a full namespace range so missing MIN or MAX means we know we have a gap issue.","commit_id":"b64dcf75b927111eaf17d9c7f4b939eeec485f45"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"072d0b9c02f723256cb617e876e33c57b078d80e","unresolved":true,"context_lines":[{"line_number":364,"context_line":"            -1 * len(path.timestamps),  # fewer timestamps"},{"line_number":365,"context_line":"            path.object_count,  # larger object count"},{"line_number":366,"context_line":"            sorted(path.timestamps)[-1]  # newest timestamp"},{"line_number":367,"context_line":"        )"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"},{"line_number":370,"context_line":"    return paths"}],"source_content_type":"text/x-python","patch_set":7,"id":"4c5b0f36_042d0af0","line":367,"updated":"2021-01-10 23:51:33.000000000","message":"I like how the we can have layers of preferences in a how to rank paths. But not 100% I understand the rankings.. maybe it\u0027s too early on a Monday morning :P\n\nWould love to discuss this somepoint this week :)","commit_id":"b64dcf75b927111eaf17d9c7f4b939eeec485f45"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05d61c2d960a171f866bf3b1f9c97410bfed1f9","unresolved":true,"context_lines":[{"line_number":382,"context_line":"            path.includes(shard_range_to_span),  # complete path for span"},{"line_number":383,"context_line":"            -1 * len(path.timestamps),  # fewer timestamps"},{"line_number":384,"context_line":"            path.object_count,  # larger object count"},{"line_number":385,"context_line":"            sorted(path.timestamps)[-1]  # newest timestamp"},{"line_number":386,"context_line":"        )"},{"line_number":387,"context_line":""},{"line_number":388,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":10,"id":"48766d60_e9a8e3fe","line":385,"updated":"2021-01-27 06:31:43.000000000","message":"I think we need to decide how this code should work. Are we planning on using this on shards. Because I\u0027d rather say we only use find_paths and rank_paths on root conatiners. If there is a split-brained shard, to be honest, who cares. Let it shard out and then let\u0027s clean it up from the root.\n\nAs such it doesn\u0027t search the full namespace (MIN - MAX) because we don\u0027t know if say MIN or MAX was missing. Oh I see now, we\u0027d just pass in the root container SR for the first order sorting (complete path for span). Cool.\n\nI\u0027m not convinced fewer timestamps or timestamps in general should be the best sorting tool for the best path. I know in Swift we tend to win on latest. But a shard (who thinks they\u0027re root because they\u0027ve been off for a while) will shard with a newer timestamp. Timestamp should be a sort, but a lower order sort in my opinon. (I do like how the latest is the last match nice touch) 😊 (Also sorry, I know some of these thoughts might not be applicable with auto-sharding turned off, but can\u0027t help it :P )\n\nI think the best determination is object count. When sharding we\u0027re talking about the amount of data to split and move. Picking the path with the most objects doesn\u0027t account for if we have a path that has just sharded or shrunk or not. Because we could end up picking a path that has in essence reverted a sharding/cleave just to have to do that work again. Or a path that has a bunch of shards that have already been collapsed. I guess if we\u0027re just looking at ACTIVE shardranges then we \"should\" only be dealing with already sharded and shrunk paths. unless we have a snaphot of a time where before the SHRUNK has made it to the root (but to have both paths here, shrunk and unshrunk, as active at the same time seems impossible).\n\nSorry just trying to work through this all in my mind. Just haven\u0027t convinced myself it isn\u0027t impossible yet :P At a minimum I think object count should come above fewer timestamps (or maybe drop fewer timestamps all together). Though i\u0027m probably missing something.\n\nAnd I\u0027m really worried about not dealing if gaps.. but I see we just drop it (in manage-shard-ranges) so that\u0027s cool. We can only focus on split-brains and come back to gaps later.. I just want to be sure the approach could be adapted for gaps rather then having to rewrite anything.\n\nI really like how this is coming along :)","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03d820c5dfb2138031cbee077c0091b71d994ee8","unresolved":true,"context_lines":[{"line_number":382,"context_line":"            path.includes(shard_range_to_span),  # complete path for span"},{"line_number":383,"context_line":"            -1 * len(path.timestamps),  # fewer timestamps"},{"line_number":384,"context_line":"            path.object_count,  # larger object count"},{"line_number":385,"context_line":"            sorted(path.timestamps)[-1]  # newest timestamp"},{"line_number":386,"context_line":"        )"},{"line_number":387,"context_line":""},{"line_number":388,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":10,"id":"127c837e_21ec64cb","line":385,"in_reply_to":"48766d60_e9a8e3fe","updated":"2021-01-27 12:33:10.000000000","message":"I\u0027m also unsure about the relative ranking of timestamps vs object count. I put fewer-timestamps first thinking it would be \u0027tidy\u0027 to preserve one half of a split-brain decision over the other by choosing a one-timestamp-path, but \u0027tidiness\u0027 may not be a good metric! 😊\n\nWe could make it a little more complex and weight the two metrics, so if object counts are not too different then fewer timestamps wins, but if object counts are very different then more objects wins.\n\nBut we could just flip to prefer object count. Without a good reason to know *why* we\u0027d make it more complex then we shouldn\u0027t.\n\nRe. Gaps - IMHO gap fixing is definitely out of scope on *this* patch. AFAIK we haven\u0027t seen gaps. But I think there is a plausible way to add gap fixing into the workflow here - find paths -\u003e *fix gappy paths* -\u003e rank paths.","commit_id":"930fa4ec93d58cd4a918b1c332f9a3f4739d136e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":392,"context_line":"            path.object_count,"},{"line_number":393,"context_line":"            # newest timestamp"},{"line_number":394,"context_line":"            sorted(path.timestamps)[-1]"},{"line_number":395,"context_line":"        )"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"},{"line_number":398,"context_line":"    return paths"}],"source_content_type":"text/x-python","patch_set":12,"id":"c583b17b_c2ed898b","line":395,"updated":"2021-02-04 06:11:24.000000000","message":"Still want to play with this. It\u0027s an awesome start and maybe good enough to always come to a valid solution, as the solution doesn\u0027t always need to be the best, but a concensous and a correct one. And maybe I sholdn\u0027t over think it.\n\nI still want to have more of a play in my SAIO","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"66da497cf07ca1e1d6ac5dd573b8cab911f6e4bb","unresolved":true,"context_lines":[{"line_number":427,"context_line":"            # most cleaving progress"},{"line_number":428,"context_line":"            path.filter_states((ShardRange.CLEAVED, ShardRange.ACTIVE)).upper,"},{"line_number":429,"context_line":"            # fewest timestamps"},{"line_number":430,"context_line":"            -1 * len(path.timestamps),"},{"line_number":431,"context_line":"            # largest object count"},{"line_number":432,"context_line":"            path.object_count,"},{"line_number":433,"context_line":"            # newest timestamp"}],"source_content_type":"text/x-python","patch_set":21,"id":"2678b0cc_c9dba2e6","line":430,"updated":"2021-03-18 04:57:54.000000000","message":"Still not sure this should be the 3rd level of sorting, maybe this could go last? or atleast further down the list.","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"11ff60d3d9217a1bd451956d051b8aac24ac02ae","unresolved":false,"context_lines":[{"line_number":427,"context_line":"            # most cleaving progress"},{"line_number":428,"context_line":"            path.filter_states((ShardRange.CLEAVED, ShardRange.ACTIVE)).upper,"},{"line_number":429,"context_line":"            # fewest timestamps"},{"line_number":430,"context_line":"            -1 * len(path.timestamps),"},{"line_number":431,"context_line":"            # largest object count"},{"line_number":432,"context_line":"            path.object_count,"},{"line_number":433,"context_line":"            # newest timestamp"}],"source_content_type":"text/x-python","patch_set":21,"id":"0f3b5771_21b03d28","line":430,"in_reply_to":"2678b0cc_c9dba2e6","updated":"2021-03-18 20:07:21.000000000","message":"Done","commit_id":"dd776b23c75bb3f6f1865ac263e479d2c44d3830"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"da65a3de3acd05cae5918bf86b655838abb3cf13","unresolved":true,"context_lines":[{"line_number":433,"context_line":"            -1 * len(path.timestamps),"},{"line_number":434,"context_line":"            # newest timestamp"},{"line_number":435,"context_line":"            sorted(path.timestamps)[-1]"},{"line_number":436,"context_line":"        )"},{"line_number":437,"context_line":""},{"line_number":438,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"},{"line_number":439,"context_line":"    return paths"}],"source_content_type":"text/x-python","patch_set":23,"id":"bc720d98_ff6200a7","line":436,"updated":"2021-03-19 05:53:35.000000000","message":"I\u0027ve been playing around and wonder if we need to take more of the threshold into account, I threw up a unit test to demonstrait: https://review.opendev.org/c/openstack/swift/+/781597\n\nBecause more timestamps are bad, in the test base I linked above, it actually would want to revert the sharded shard (which is the difference), meaning we\u0027d need to shard all over again.\n\nHappy for this to be a first pass though, because it\u0027s otherwise pretty cool way to choose by different priorities. I\u0027ll keep playing.\n\nMaybe in a follow up, I could dust off my weigting algoritm to see how it\u0027ll go as one of the ranks above (the one the neg effects small and over container threshold shards, but otherwise but otherwise bases it of obj count) as it might be able to simply drop it as a replacement for path.object on line 431.\n\nUPDATE: something like the calculate_range_weight in https://review.opendev.org/c/openstack/swift/+/749614/7/swift/container/sharder.py","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a60ef8d4483109ca3e7fe8f54696087dd2bc7732","unresolved":true,"context_lines":[{"line_number":433,"context_line":"            -1 * len(path.timestamps),"},{"line_number":434,"context_line":"            # newest timestamp"},{"line_number":435,"context_line":"            sorted(path.timestamps)[-1]"},{"line_number":436,"context_line":"        )"},{"line_number":437,"context_line":""},{"line_number":438,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"},{"line_number":439,"context_line":"    return paths"}],"source_content_type":"text/x-python","patch_set":23,"id":"877b4dab_7bc45196","line":436,"in_reply_to":"bc720d98_ff6200a7","updated":"2021-03-22 18:50:47.000000000","message":"ok, I see your point, maybe we can do better. Hard thing is going to be if we need to know the expected object counts *after* the repair is applied, because we cannot predict that with overlaps. But we could try to avoid already large shards.","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"531d6ac2f559771690c2c37be1edc7f66f7d2b52","unresolved":true,"context_lines":[{"line_number":378,"context_line":"            # the CalledProcessError wasn\u0027t caught by that! despite type(exc)"},{"line_number":379,"context_line":"            # being a CalledProcessError, isinstance(exc, CalledProcessError)"},{"line_number":380,"context_line":"            # is False and the type has a different hash - perhaps something to"},{"line_number":381,"context_line":"            # do with random PYTHONHASHSEEDs???"},{"line_number":382,"context_line":"            try:"},{"line_number":383,"context_line":"                # assume this is a CalledProcessError"},{"line_number":384,"context_line":"                self.fail(\u0027%s with output:\\n%s\u0027 % (exc, exc.output))"}],"source_content_type":"text/x-python","patch_set":5,"id":"53aca86c_3e718142","line":381,"updated":"2020-12-11 15:21:11.000000000","message":"this is annoying and I\u0027d appreciate any suggestions","commit_id":"51418bdd6644a11e56027e0728baa28b0c66bfe4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":372,"context_line":"            # the CalledProcessError wasn\u0027t caught by that! despite type(exc)"},{"line_number":373,"context_line":"            # being a CalledProcessError, isinstance(exc, CalledProcessError)"},{"line_number":374,"context_line":"            # is False and the type has a different hash - could be"},{"line_number":375,"context_line":"            # related to https://github.com/eventlet/eventlet/issues/413"},{"line_number":376,"context_line":"            try:"},{"line_number":377,"context_line":"                # assume this is a CalledProcessError"},{"line_number":378,"context_line":"                self.fail(\u0027%s with output:\\n%s\u0027 % (exc, exc.output))"}],"source_content_type":"text/x-python","patch_set":12,"id":"c415a457_24128602","line":375,"updated":"2021-02-04 06:11:24.000000000","message":"yikes, sounds like you\u0027ve had some \"fun\" :P","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"26d9086fe588b760b636a2ba1d0f28f370d0c4dc","unresolved":true,"context_lines":[{"line_number":372,"context_line":"            # the CalledProcessError wasn\u0027t caught by that! despite type(exc)"},{"line_number":373,"context_line":"            # being a CalledProcessError, isinstance(exc, CalledProcessError)"},{"line_number":374,"context_line":"            # is False and the type has a different hash - could be"},{"line_number":375,"context_line":"            # related to https://github.com/eventlet/eventlet/issues/413"},{"line_number":376,"context_line":"            try:"},{"line_number":377,"context_line":"                # assume this is a CalledProcessError"},{"line_number":378,"context_line":"                self.fail(\u0027%s with output:\\n%s\u0027 % (exc, exc.output))"}],"source_content_type":"text/x-python","patch_set":12,"id":"a89f1dea_c0bfc93b","line":375,"in_reply_to":"c415a457_24128602","updated":"2021-02-04 19:13:58.000000000","message":"yep, that was baffling","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"}],"test/unit/cli/test_manage_shard_ranges.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"da65a3de3acd05cae5918bf86b655838abb3cf13","unresolved":true,"context_lines":[{"line_number":1278,"context_line":"        self.assert_shard_ranges_equal([], updated_ranges)"},{"line_number":1279,"context_line":""},{"line_number":1280,"context_line":"    def test_repair_gaps(self):"},{"line_number":1281,"context_line":"        # once incomplete sequence"},{"line_number":1282,"context_line":"        broker \u003d self._make_broker()"},{"line_number":1283,"context_line":"        broker.set_sharding_sysmeta(\u0027Quoted-Root\u0027, \u0027a/c\u0027)"},{"line_number":1284,"context_line":"        with mock_timestamp_now(next(self.ts_iter)):"}],"source_content_type":"text/x-python","patch_set":23,"id":"242513d7_46d90cee","line":1281,"range":{"start_line":1281,"start_character":10,"end_line":1281,"end_character":14},"updated":"2021-03-19 05:53:35.000000000","message":"one","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"25f493c00529fa04d42fba234f97610515595f88","unresolved":false,"context_lines":[{"line_number":1278,"context_line":"        self.assert_shard_ranges_equal([], updated_ranges)"},{"line_number":1279,"context_line":""},{"line_number":1280,"context_line":"    def test_repair_gaps(self):"},{"line_number":1281,"context_line":"        # once incomplete sequence"},{"line_number":1282,"context_line":"        broker \u003d self._make_broker()"},{"line_number":1283,"context_line":"        broker.set_sharding_sysmeta(\u0027Quoted-Root\u0027, \u0027a/c\u0027)"},{"line_number":1284,"context_line":"        with mock_timestamp_now(next(self.ts_iter)):"}],"source_content_type":"text/x-python","patch_set":23,"id":"dbe7112f_ad998958","line":1281,"range":{"start_line":1281,"start_character":10,"end_line":1281,"end_character":14},"in_reply_to":"242513d7_46d90cee","updated":"2021-03-25 12:54:07.000000000","message":"Done","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"da65a3de3acd05cae5918bf86b655838abb3cf13","unresolved":true,"context_lines":[{"line_number":1306,"context_line":"                broker,"},{"line_number":1307,"context_line":"                self.overlap_shard_data_1[:2] + self.overlap_shard_data_1[6:],"},{"line_number":1308,"context_line":"                \u0027.shards_\u0027)"},{"line_number":1309,"context_line":"        broker.merge_shard_ranges(overlap_shard_ranges)"},{"line_number":1310,"context_line":"        out \u003d StringIO()"},{"line_number":1311,"context_line":"        err \u003d StringIO()"},{"line_number":1312,"context_line":"        with mock.patch(\u0027sys.stdout\u0027, out), mock.patch(\u0027sys.stderr\u0027, err):"}],"source_content_type":"text/x-python","patch_set":23,"id":"a97709b4_2c1b176d","line":1309,"updated":"2021-03-19 05:53:35.000000000","message":"Would this be 2 incomplete incomplete or 3? The shard_data without the last range, and then 2 parts of the overlap_shard_data?","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a60ef8d4483109ca3e7fe8f54696087dd2bc7732","unresolved":true,"context_lines":[{"line_number":1306,"context_line":"                broker,"},{"line_number":1307,"context_line":"                self.overlap_shard_data_1[:2] + self.overlap_shard_data_1[6:],"},{"line_number":1308,"context_line":"                \u0027.shards_\u0027)"},{"line_number":1309,"context_line":"        broker.merge_shard_ranges(overlap_shard_ranges)"},{"line_number":1310,"context_line":"        out \u003d StringIO()"},{"line_number":1311,"context_line":"        err \u003d StringIO()"},{"line_number":1312,"context_line":"        with mock.patch(\u0027sys.stdout\u0027, out), mock.patch(\u0027sys.stderr\u0027, err):"}],"source_content_type":"text/x-python","patch_set":23,"id":"dfccbcdc_ce3dd34b","line":1309,"in_reply_to":"a97709b4_2c1b176d","updated":"2021-03-22 18:50:47.000000000","message":"oh, I didn\u0027t reset the broker :(","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"25f493c00529fa04d42fba234f97610515595f88","unresolved":false,"context_lines":[{"line_number":1306,"context_line":"                broker,"},{"line_number":1307,"context_line":"                self.overlap_shard_data_1[:2] + self.overlap_shard_data_1[6:],"},{"line_number":1308,"context_line":"                \u0027.shards_\u0027)"},{"line_number":1309,"context_line":"        broker.merge_shard_ranges(overlap_shard_ranges)"},{"line_number":1310,"context_line":"        out \u003d StringIO()"},{"line_number":1311,"context_line":"        err \u003d StringIO()"},{"line_number":1312,"context_line":"        with mock.patch(\u0027sys.stdout\u0027, out), mock.patch(\u0027sys.stderr\u0027, err):"}],"source_content_type":"text/x-python","patch_set":23,"id":"50b53734_4fa042e6","line":1309,"in_reply_to":"dfccbcdc_ce3dd34b","updated":"2021-03-25 12:54:07.000000000","message":"Done","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"da65a3de3acd05cae5918bf86b655838abb3cf13","unresolved":true,"context_lines":[{"line_number":1422,"context_line":"            key\u003dShardRange.sort_key)"},{"line_number":1423,"context_line":"        self.assert_shard_ranges_equal(expected, updated_ranges)"},{"line_number":1424,"context_line":""},{"line_number":1425,"context_line":"        # user input \u0027y\u0027"},{"line_number":1426,"context_line":"        ts_now \u003d next(self.ts_iter)"},{"line_number":1427,"context_line":"        do_repair(\u0027yes\u0027, ts_now)"},{"line_number":1428,"context_line":"        updated_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":23,"id":"8aba2aa9_7de0b167","line":1425,"range":{"start_line":1425,"start_character":22,"end_line":1425,"end_character":23},"updated":"2021-03-19 05:53:35.000000000","message":"yes","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a60ef8d4483109ca3e7fe8f54696087dd2bc7732","unresolved":true,"context_lines":[{"line_number":1422,"context_line":"            key\u003dShardRange.sort_key)"},{"line_number":1423,"context_line":"        self.assert_shard_ranges_equal(expected, updated_ranges)"},{"line_number":1424,"context_line":""},{"line_number":1425,"context_line":"        # user input \u0027y\u0027"},{"line_number":1426,"context_line":"        ts_now \u003d next(self.ts_iter)"},{"line_number":1427,"context_line":"        do_repair(\u0027yes\u0027, ts_now)"},{"line_number":1428,"context_line":"        updated_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":23,"id":"d700ba47_eb68004e","line":1425,"range":{"start_line":1425,"start_character":22,"end_line":1425,"end_character":23},"in_reply_to":"8aba2aa9_7de0b167","updated":"2021-03-22 18:50:47.000000000","message":"yes! I get confused because compact requires \u0027y\u0027 - somewhere else I fix things so that repair and compact both require \u0027yes\u0027","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"25f493c00529fa04d42fba234f97610515595f88","unresolved":false,"context_lines":[{"line_number":1422,"context_line":"            key\u003dShardRange.sort_key)"},{"line_number":1423,"context_line":"        self.assert_shard_ranges_equal(expected, updated_ranges)"},{"line_number":1424,"context_line":""},{"line_number":1425,"context_line":"        # user input \u0027y\u0027"},{"line_number":1426,"context_line":"        ts_now \u003d next(self.ts_iter)"},{"line_number":1427,"context_line":"        do_repair(\u0027yes\u0027, ts_now)"},{"line_number":1428,"context_line":"        updated_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":23,"id":"463eb1b3_0d6cc44e","line":1425,"range":{"start_line":1425,"start_character":22,"end_line":1425,"end_character":23},"in_reply_to":"d700ba47_eb68004e","updated":"2021-03-25 12:54:07.000000000","message":"Done","commit_id":"a9757c879d4f7868a9e3e57ce720087af875f186"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa1d7533af40f88a5a68057ee576466f110e39be","unresolved":true,"context_lines":[{"line_number":6063,"context_line":"                ranges_0,  # cleaved to m"},{"line_number":6064,"context_line":"            ],"},{"line_number":6065,"context_line":"            rank_paths(paths, own_sr)"},{"line_number":6066,"context_line":"        )"}],"source_content_type":"text/x-python","patch_set":12,"id":"9f4fc432_c099f75a","line":6066,"updated":"2021-02-04 06:11:24.000000000","message":"Nice, I might go find some of my POC mucked up shardranges and give it a whirl with your find and rank paths code! But that will have to be tomorrow as I\u0027m running out of time today.","commit_id":"84dfe09d84092de07bbd82798bb127cf0f2311a5"}]}