)]}'
{"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":300,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":301,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":302,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":303,"context_line":"    cxt \u003d CleavingContext.load(broker)"},{"line_number":304,"context_line":"    # want to start scanning again from the beginning."},{"line_number":305,"context_line":"    if cxt.scan_progress:"},{"line_number":306,"context_line":"        cxt.scan_progress \u003d 0"}],"source_content_type":"text/x-python","patch_set":4,"id":"c7a38714_1ee272ff","line":303,"range":{"start_line":303,"start_character":4,"end_line":303,"end_character":8},"updated":"2021-06-02 12:06:26.000000000","message":"nit: typically \u0027context\u0027 gets shortened to ctx or ctxt","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4523258ce203971e535c2939aca83995d7fc0347","unresolved":true,"context_lines":[{"line_number":300,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":301,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":302,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":303,"context_line":"    cxt \u003d CleavingContext.load(broker)"},{"line_number":304,"context_line":"    # want to start scanning again from the beginning."},{"line_number":305,"context_line":"    if cxt.scan_progress:"},{"line_number":306,"context_line":"        cxt.scan_progress \u003d 0"}],"source_content_type":"text/x-python","patch_set":4,"id":"493f5236_10adb715","line":303,"range":{"start_line":303,"start_character":4,"end_line":303,"end_character":8},"in_reply_to":"c7a38714_1ee272ff","updated":"2021-06-03 08:12:24.000000000","message":"yeah fair enough, I wondered but after a while the 3 letters (so long as they start with c) look the same 😋","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":321,"context_line":"                                         progress\u003dcxt.scan_progress,"},{"line_number":322,"context_line":"                                         minimum_size\u003dargs.minimum_shard_size)"},{"line_number":323,"context_line":"            shard_data.extend(more_shard_data)"},{"line_number":324,"context_line":"        cxt.store(broker)"},{"line_number":325,"context_line":"    return shard_data, time.time() - start"},{"line_number":326,"context_line":""},{"line_number":327,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"047e0d8a_1d63b622","line":324,"range":{"start_line":324,"start_character":8,"end_line":324,"end_character":25},"updated":"2021-06-02 12:06:26.000000000","message":"ok, so we\u0027re adding a cleaving context earlier in the sharding process than before - is there anywhere that the mere *existence* of a cleaving context in the broker has significance?","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4523258ce203971e535c2939aca83995d7fc0347","unresolved":true,"context_lines":[{"line_number":321,"context_line":"                                         progress\u003dcxt.scan_progress,"},{"line_number":322,"context_line":"                                         minimum_size\u003dargs.minimum_shard_size)"},{"line_number":323,"context_line":"            shard_data.extend(more_shard_data)"},{"line_number":324,"context_line":"        cxt.store(broker)"},{"line_number":325,"context_line":"    return shard_data, time.time() - start"},{"line_number":326,"context_line":""},{"line_number":327,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"77683d33_be0cf4e7","line":324,"range":{"start_line":324,"start_character":8,"end_line":324,"end_character":25},"in_reply_to":"047e0d8a_1d63b622","updated":"2021-06-03 08:12:24.000000000","message":"I don\u0027t think so. We only use them when it gets time to sharding or checking for completeness etc. There is a cleavecontext audit in the sharder, I guess we should confirm.","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"}],"swift/container/backend.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1593bc147b66256aaafef3317c23d9ca8f7bda6f","unresolved":true,"context_lines":[{"line_number":2316,"context_line":"                # it does, mark it as zero and let the shard cycle deal with"},{"line_number":2317,"context_line":"                # what the correct object count would be."},{"line_number":2318,"context_line":"                if shard_size \u003c 0:"},{"line_number":2319,"context_line":"                    shard_size \u003d 0"},{"line_number":2320,"context_line":""},{"line_number":2321,"context_line":"            # NB shard ranges are created with a non-zero object count so that"},{"line_number":2322,"context_line":"            # the apparent container object count remains constant, and the"}],"source_content_type":"text/x-python","patch_set":2,"id":"353003c0_d441534c","line":2319,"updated":"2021-05-31 10:13:19.000000000","message":"OK so this change simplifys this code.. and maybe not for the better. But if we\u0027re going to be tracking progress in the CleaveContext then if we get to a place where we\u0027re at the \"end\" earlier or beyond the end, then marking at zero I think makes sense. The shardrange will be FOUND and then CLEAVED which by that stage it\u0027ll have it\u0027s true object_count.. if it happens to be huge once it becomes ACTIVE it\u0027ll shard again (which I think is the correct action).\n\nIt may come back small and I guess we just have ourselves a small tail.\n\nWe could just push forward and search try and get the next upper and continue, that\u0027s the other option.. but if we fall back to that, then we\u0027re back and small tails. And the whole point is to track this containers progress.\n\nReal question then becomes. What happens if there is a rebalance.. I guess the new scanner looses it\u0027s progress.. so maybe we need to fall back to an imperfect, calculate progress from already found shards... if they are alittle out of whack, then worst case my first paragraph of this comment comes back into play. And of course, that only effects auto-sharding not s-m-s-r which will scan in batches but on the same replica until it\u0027s done.","commit_id":"31e61908070a8f693b8897ce30db29328293f234"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a1afa7508e0a67ada6be8c42869e4d94d2ad1e83","unresolved":true,"context_lines":[{"line_number":2316,"context_line":"                # it does, mark it as zero and let the shard cycle deal with"},{"line_number":2317,"context_line":"                # what the correct object count would be."},{"line_number":2318,"context_line":"                if shard_size \u003c 0:"},{"line_number":2319,"context_line":"                    shard_size \u003d 0"},{"line_number":2320,"context_line":""},{"line_number":2321,"context_line":"            # NB shard ranges are created with a non-zero object count so that"},{"line_number":2322,"context_line":"            # the apparent container object count remains constant, and the"}],"source_content_type":"text/x-python","patch_set":2,"id":"c5914b37_ec1f6cec","line":2319,"in_reply_to":"353003c0_d441534c","updated":"2021-05-31 10:24:30.000000000","message":"Maybe something like http://paste.openstack.org/show/805857/","commit_id":"31e61908070a8f693b8897ce30db29328293f234"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":2250,"context_line":"        :param shard_size: the size of each shard range"},{"line_number":2251,"context_line":"        :param limit: the maximum number of shard points to be found; a"},{"line_number":2252,"context_line":"            negative value (default) implies no limit."},{"line_number":2253,"context_line":"        :param existing_ranges: an optional list of existing ShardRanges; if"},{"line_number":2254,"context_line":"            given, this list should be sorted in order of upper bounds; the"},{"line_number":2255,"context_line":"            scan for new shard ranges will start at the upper bound of the last"},{"line_number":2256,"context_line":"            existing ShardRange."}],"source_content_type":"text/x-python","patch_set":4,"id":"f7dfa789_d7430168","line":2253,"range":{"start_line":2253,"start_character":15,"end_line":2253,"end_character":30},"updated":"2021-06-02 12:06:26.000000000","message":"we use this to get the last_upper and possibly to get the progress if not given...could we change the signature to just pass in last_upper and progress? in fact, could we just pass in a context and have this method mutate ctx.progress and ctx.scan_cursor??? \n\nIDK which interface I prefer, so will mull on it for a while. But something about falling back to existing_ranges if progress is not given seems more complicated than necessary.","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":2276,"context_line":"        if existing_ranges:"},{"line_number":2277,"context_line":"            last_shard_upper \u003d existing_ranges[-1].upper"},{"line_number":2278,"context_line":"            if progress \u003d\u003d 0:"},{"line_number":2279,"context_line":"                progress \u003d sum([sr.object_count for sr in existing_ranges])"},{"line_number":2280,"context_line":"            if last_shard_upper \u003e\u003d own_shard_range.upper:"},{"line_number":2281,"context_line":"                # \u003d\u003d implies all ranges were previously found"},{"line_number":2282,"context_line":"                # \u003e implies an acceptor range has been set into which this"}],"source_content_type":"text/x-python","patch_set":4,"id":"f2be4048_110f45d1","line":2279,"updated":"2021-06-02 12:06:26.000000000","message":"here we\u0027re still using the potentially unreliable progress based on existing ranges, but without the check on states\n\nShardRangeList provides an object count summation, wonder if we could use that?","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c070977dc5af75fd3712bcda37c0ef7ae438227a","unresolved":true,"context_lines":[{"line_number":2276,"context_line":"        if existing_ranges:"},{"line_number":2277,"context_line":"            last_shard_upper \u003d existing_ranges[-1].upper"},{"line_number":2278,"context_line":"            if progress \u003d\u003d 0:"},{"line_number":2279,"context_line":"                progress \u003d sum([sr.object_count for sr in existing_ranges])"},{"line_number":2280,"context_line":"            if last_shard_upper \u003e\u003d own_shard_range.upper:"},{"line_number":2281,"context_line":"                # \u003d\u003d implies all ranges were previously found"},{"line_number":2282,"context_line":"                # \u003e implies an acceptor range has been set into which this"}],"source_content_type":"text/x-python","patch_set":4,"id":"62b8d3a7_3db087fb","line":2279,"in_reply_to":"f2be4048_110f45d1","updated":"2021-06-03 06:57:14.000000000","message":"I wonder if we should do something like: http://paste.openstack.org/show/806300/\n\nSo it\u0027ll hit the DB to get a reliable progress, but this should only happen rarely if ever, but would mean we don\u0027t have cater to the non-reiable method.","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":2289,"context_line":"        sub_broker \u003d self.get_brokers()[0]"},{"line_number":2290,"context_line":"        index \u003d len(existing_ranges)"},{"line_number":2291,"context_line":"        while limit is None or limit \u003c 0 or len(found_ranges) \u003c limit:"},{"line_number":2292,"context_line":"            if progress + shard_size + minimum_size \u003e\u003d object_count:"},{"line_number":2293,"context_line":"                # next shard point is at or beyond final object name so don\u0027t"},{"line_number":2294,"context_line":"                # bother with db query"},{"line_number":2295,"context_line":"                next_shard_upper \u003d None"}],"source_content_type":"text/x-python","patch_set":4,"id":"23967c65_d1a21348","line":2292,"range":{"start_line":2292,"start_character":15,"end_line":2292,"end_character":67},"updated":"2021-06-02 12:06:26.000000000","message":"ok, this works and is simple\n\nTracking progress independently of existing range object counts is an improvement.\n\nHowever, this is still a \u0027look ahead and squash in next tiny range\u0027 approach. It is based on a potentially inaccurate estimate of the shard size because it is based on comparing object_count to progress. IF we decide to make the estimate more accurate we\u0027d need to make a look ahead db query here (which gets back to the \u0027wasted db query\u0027 issue discussed on other patches).\n\nIf we do the minimum size check and squash in the caller, rather than here, then IF we decide to improve the accuracy of the final shard size using a db query then I suspect it will be much less disruptive. i.e. this method always returns the shard range it finds, with an estimate or accurate shard size, then caller decides whether to squash the final shard into the penultimate.\n\nBut, maybe we won\u0027t ever make it more accurate :/","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"db2cfa127e9d4a79d56d13c45a354acd5d862bb3","unresolved":true,"context_lines":[{"line_number":1482,"context_line":"        shard_data, last_found, cxt.scan_progress \u003d broker.find_shard_ranges("},{"line_number":1483,"context_line":"            self.rows_per_shard, limit\u003dself.shard_scanner_batch_size,"},{"line_number":1484,"context_line":"            existing_ranges\u003dshard_ranges, progress\u003dcxt.scan_progress,"},{"line_number":1485,"context_line":"            minimum_size\u003dself.minimum_shard_size)"},{"line_number":1486,"context_line":"        cxt.store(broker)"},{"line_number":1487,"context_line":"        elapsed \u003d time.time() - start"},{"line_number":1488,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"fff90c6f_e597536a","line":1485,"updated":"2021-06-02 12:06:26.000000000","message":"in this we need to remember upgrade - there might be an autosharding container that has found *some* ranges but does *not* have a context. After upgrade, the context should take the progress of the existing ranges.","commit_id":"366af3e24ae5b8142f940b78feb43d3ca45630cd"}]}