)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":17,"context_line":"Further, when we find and create a shard, it starts out with the number"},{"line_number":18,"context_line":"of objects, at the time, that are expected to go into them. This is"},{"line_number":19,"context_line":"better answer then, say, nothing."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"So it\u0027s better for the shard to send it\u0027s current own_shard_range but"},{"line_number":22,"context_line":"don\u0027t update the stats until it can be authoritive of that answer."},{"line_number":23,"context_line":"This patch adds a new SHARD_UPDATE_STAT_STATES that track what"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"94e2aa2b_dea5fbe0","line":20,"updated":"2022-12-09 11:49:28.000000000","message":"nit: needs line-wrapping","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":17,"context_line":"Further, when we find and create a shard, it starts out with the number"},{"line_number":18,"context_line":"of objects, at the time, that are expected to go into them. This is"},{"line_number":19,"context_line":"better answer then, say, nothing."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"So it\u0027s better for the shard to send it\u0027s current own_shard_range but"},{"line_number":22,"context_line":"don\u0027t update the stats until it can be authoritive of that answer."},{"line_number":23,"context_line":"This patch adds a new SHARD_UPDATE_STAT_STATES that track what"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"899dfba6_4699646f","line":20,"in_reply_to":"94e2aa2b_dea5fbe0","updated":"2022-12-12 06:42:13.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":28,"context_line":"                                ShardRange.SHARDING, ShardRange.SHARDED,"},{"line_number":29,"context_line":"                                ShardRange.SHRINKING, ShardRange.SHRUNK]"},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"This also has the advantage of a cleaving shard to have an idea of the"},{"line_number":32,"context_line":"number of objects expected to be cleaved into it, at least from the"},{"line_number":33,"context_line":"leader who split the namespace is concerned."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"This could be used to allow a small replica to know if it has an"},{"line_number":36,"context_line":"undersized portion of the required namespace and so wait until the other"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"8b3bd10b_93732a39","line":33,"range":{"start_line":31,"start_character":1,"end_line":33,"end_character":44},"updated":"2022-12-09 11:49:28.000000000","message":"this patch doesn\u0027t change the cleaving behavior, and a shard doesn\u0027t ever need to know how many rows to expect to be cleaved into it, I think?","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":28,"context_line":"                                ShardRange.SHARDING, ShardRange.SHARDED,"},{"line_number":29,"context_line":"                                ShardRange.SHRINKING, ShardRange.SHRUNK]"},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"This also has the advantage of a cleaving shard to have an idea of the"},{"line_number":32,"context_line":"number of objects expected to be cleaved into it, at least from the"},{"line_number":33,"context_line":"leader who split the namespace is concerned."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"This could be used to allow a small replica to know if it has an"},{"line_number":36,"context_line":"undersized portion of the required namespace and so wait until the other"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"772b6d4e_17702438","line":33,"range":{"start_line":31,"start_character":1,"end_line":33,"end_character":44},"in_reply_to":"8b3bd10b_93732a39","updated":"2022-12-12 06:42:13.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":32,"context_line":"number of objects expected to be cleaved into it, at least from the"},{"line_number":33,"context_line":"leader who split the namespace is concerned."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"This could be used to allow a small replica to know if it has an"},{"line_number":36,"context_line":"undersized portion of the required namespace and so wait until the other"},{"line_number":37,"context_line":"replicas to cleave before moving on."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As we don\u0027t want to update the OSR stats and the meta_timestmap, also"},{"line_number":40,"context_line":"move tombstone updates to only happen when in a SHARD_UPDATE_STAT_STATES"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"0ad0d1c2_9b568f99","line":37,"range":{"start_line":35,"start_character":1,"end_line":37,"end_character":36},"updated":"2022-12-09 11:49:28.000000000","message":"this is the next patch I think. I\u0027d suggest removing this and the previous paragraph from this commit message.\n\nI think it is sufficient that this patch fixes anomalous/inaccurate stats updates while a shard is in CREATED state - that in itself justifies the patch.","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":32,"context_line":"number of objects expected to be cleaved into it, at least from the"},{"line_number":33,"context_line":"leader who split the namespace is concerned."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"This could be used to allow a small replica to know if it has an"},{"line_number":36,"context_line":"undersized portion of the required namespace and so wait until the other"},{"line_number":37,"context_line":"replicas to cleave before moving on."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As we don\u0027t want to update the OSR stats and the meta_timestmap, also"},{"line_number":40,"context_line":"move tombstone updates to only happen when in a SHARD_UPDATE_STAT_STATES"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"551de869_0f01ebe2","line":37,"range":{"start_line":35,"start_character":1,"end_line":37,"end_character":36},"in_reply_to":"0ad0d1c2_9b568f99","updated":"2022-12-12 06:42:13.000000000","message":"yeah, these comments are obviously jumping the gun a little. Will pull it back into what this change actually does.","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a5f46d9491d1a08048f840a0eacc0f2703f1a3a6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"89b0645e_34ae425d","updated":"2022-08-03 07:37:57.000000000","message":"New patchset incoming","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"8635ae1e_dfc63a87","updated":"2022-08-05 17:52:43.000000000","message":"@Matt I think the principle of this change is right, but I\u0027d like to explore moving away from the confusing semantics of get_own_shard_range - see https://review.opendev.org/c/openstack/swift/+/852283 (only had time to push up a quick WIP, but could be a base for this change if we like it)","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dbac0b989a6bfafe0dd616b3e5f5095f96f0db99","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"0d7ef51c_9135eb83","updated":"2022-12-08 18:07:10.000000000","message":"@Matt - I started looking at this again, and ending up hacking some tests to help me understand https://review.opendev.org/c/openstack/swift/+/867038/1\n\nTook me a while to remember that this patch alone does not solve the underpopulated cleaved shard problem, because the sharder still sets actual object stats when a shard reaches CLEAVED - we need the next patch to fix that. But this patch lays the ground for off-whack stats being reported before CLEAVED (which is perhaps an existing bug/non-feature in itself), which my probe test change demonstrates.","commit_id":"70e57f8059c28b43934323bde1f44f8685ddda11"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"51a38fed_19a10fbe","updated":"2022-12-09 11:49:28.000000000","message":"This change makes sense to me - I think it fixes a window of anomalous stats reporting which is in itself a good thing, and needed to be corrected before we can work more on the transition to CLEAVING.\n\nI think it\u0027s worth a little more polishing - see comments and diff. Thanks Matt!","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ccd0f2672102dbf45c5399e43fa70be7fdc0c1a","unresolved":true,"context_lines":[{"line_number":62,"context_line":"                         ShardRange.SHARDED, ShardRange.SHRINKING,"},{"line_number":63,"context_line":"                         ShardRange.SHRUNK]"},{"line_number":64,"context_line":"SHARD_UPDATE_STAT_STATES \u003d SHARD_LISTING_STATES + [ShardRange.SHARDED,"},{"line_number":65,"context_line":"                                                   ShardRange.SHRUNK]"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"# attribute names in order used when transforming shard ranges from dicts to"},{"line_number":68,"context_line":"# tuples and vice-versa"}],"source_content_type":"text/x-python","patch_set":2,"id":"a69b1a95_5d38ca78","line":65,"updated":"2022-08-03 05:03:05.000000000","message":"Might be worth spelling these all out, like we did for SHARD_AUDITING_STATES. Not sure whether it was the double-hop of SHARD_STATS_STATES -\u003e SHARD_LISTING_STATES -\u003e SHARD_UPDATE_STAT_STATES, the distance from the other definitions, or just my own tiredness, but it took me several read-throughs to see that this included SHARDING and SHRINKING.\n\nThe thrust of it is, \"everything except FOUND and CREATED\", right? And even FOUND is only sort of excluded, since there should never be an actual DB in FOUND state, right?","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a5f46d9491d1a08048f840a0eacc0f2703f1a3a6","unresolved":false,"context_lines":[{"line_number":62,"context_line":"                         ShardRange.SHARDED, ShardRange.SHRINKING,"},{"line_number":63,"context_line":"                         ShardRange.SHRUNK]"},{"line_number":64,"context_line":"SHARD_UPDATE_STAT_STATES \u003d SHARD_LISTING_STATES + [ShardRange.SHARDED,"},{"line_number":65,"context_line":"                                                   ShardRange.SHRUNK]"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"# attribute names in order used when transforming shard ranges from dicts to"},{"line_number":68,"context_line":"# tuples and vice-versa"}],"source_content_type":"text/x-python","patch_set":2,"id":"be426b13_b2c31610","line":65,"in_reply_to":"a69b1a95_5d38ca78","updated":"2022-08-03 07:37:57.000000000","message":"Yeah, I can understand that, and it is getting a little unwieldy. Specifying it fully probably is a better and easier maintainable way. When the other \"states\" dont really releate the update_stats. Will change.","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ccd0f2672102dbf45c5399e43fa70be7fdc0c1a","unresolved":true,"context_lines":[{"line_number":2344,"context_line":"                    # object_count"},{"line_number":2345,"context_line":"                    shard_size \u003d object_count - progress"},{"line_number":2346,"context_line":""},{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"}],"source_content_type":"text/x-python","patch_set":2,"id":"ed6a8875_5b8db304","line":2347,"range":{"start_line":2347,"start_character":58,"end_line":2347,"end_character":70},"updated":"2022-08-03 05:03:05.000000000","message":"And byte count!","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":2344,"context_line":"                    # object_count"},{"line_number":2345,"context_line":"                    shard_size \u003d object_count - progress"},{"line_number":2346,"context_line":""},{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"}],"source_content_type":"text/x-python","patch_set":2,"id":"d9067d79_7b27e393","line":2347,"range":{"start_line":2347,"start_character":58,"end_line":2347,"end_character":70},"in_reply_to":"64c18452_4ad75e5e","updated":"2022-12-09 11:49:28.000000000","message":"I don\u0027t think we even know the byte count from the found shard ranges, correct?","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":2344,"context_line":"                    # object_count"},{"line_number":2345,"context_line":"                    shard_size \u003d object_count - progress"},{"line_number":2346,"context_line":""},{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"}],"source_content_type":"text/x-python","patch_set":2,"id":"73bf0613_33d6b0e0","line":2347,"range":{"start_line":2347,"start_character":58,"end_line":2347,"end_character":70},"in_reply_to":"d9067d79_7b27e393","updated":"2022-12-12 06:42:13.000000000","message":"correct, we are only going on object count. Because we are basically sharding the object list, where byte count isn\u0027t important, at least from the sharding POV.\n\nI guess we could start tracking it, it may give us some new features. But when finding the shards we tend to jump to the next upper so we don\u0027t have to process each row. So doesn\u0027t really make sense when finding ranges, unless in the future we decide to shard on amount of space they take up on the object server side of things, which would more be a placement thing not a sharding object metadata thing. In anycase I think out of scope of this change.","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a5f46d9491d1a08048f840a0eacc0f2703f1a3a6","unresolved":true,"context_lines":[{"line_number":2344,"context_line":"                    # object_count"},{"line_number":2345,"context_line":"                    shard_size \u003d object_count - progress"},{"line_number":2346,"context_line":""},{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"}],"source_content_type":"text/x-python","patch_set":2,"id":"64c18452_4ad75e5e","line":2347,"range":{"start_line":2347,"start_character":58,"end_line":2347,"end_character":70},"in_reply_to":"ed6a8875_5b8db304","updated":"2022-08-03 07:37:57.000000000","message":"We don\u0027t actually pull in the byte-count for each range we keep them 0. But its the object count that we use for sharding and is_deleted checks so needs to remain. Once the range is CLEAVED it should have a more accurate bytes_count, so I think it\u0027s all ok.","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ccd0f2672102dbf45c5399e43fa70be7fdc0c1a","unresolved":true,"context_lines":[{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"},{"line_number":2351,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2352,"context_line":"            #     but not yet cleaved; and"},{"line_number":2353,"context_line":"            #  3. So we have an rough idea the size of the shards should be"}],"source_content_type":"text/x-python","patch_set":2,"id":"8aef6096_d49b8af1","line":2350,"range":{"start_line":2350,"start_character":18,"end_line":2350,"end_character":26},"updated":"2022-08-03 05:03:05.000000000","message":"Off-topic: \"constant\" isn\u0027t quite right, but I\u0027m not sure how better to put it. At any rate, we don\u0027t want it flapping about wildly.","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a5f46d9491d1a08048f840a0eacc0f2703f1a3a6","unresolved":true,"context_lines":[{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"},{"line_number":2351,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2352,"context_line":"            #     but not yet cleaved; and"},{"line_number":2353,"context_line":"            #  3. So we have an rough idea the size of the shards should be"}],"source_content_type":"text/x-python","patch_set":2,"id":"a8a9f3bc_95219278","line":2350,"range":{"start_line":2350,"start_character":18,"end_line":2350,"end_character":26},"in_reply_to":"8aef6096_d49b8af1","updated":"2022-08-03 07:37:57.000000000","message":"representative? consistent? I\u0027ll pick something else :)","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":false,"context_lines":[{"line_number":2347,"context_line":"            # NB shard ranges are created with a non-zero object count for a"},{"line_number":2348,"context_line":"            # few reasons:"},{"line_number":2349,"context_line":"            #  1. so that the apparent container object count remains"},{"line_number":2350,"context_line":"            #     constant;"},{"line_number":2351,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2352,"context_line":"            #     but not yet cleaved; and"},{"line_number":2353,"context_line":"            #  3. So we have an rough idea the size of the shards should be"}],"source_content_type":"text/x-python","patch_set":2,"id":"2cac1182_07da1d3a","line":2350,"range":{"start_line":2350,"start_character":18,"end_line":2350,"end_character":26},"in_reply_to":"a8a9f3bc_95219278","updated":"2022-12-09 11:49:28.000000000","message":"Done","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ccd0f2672102dbf45c5399e43fa70be7fdc0c1a","unresolved":true,"context_lines":[{"line_number":2350,"context_line":"            #     constant;"},{"line_number":2351,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2352,"context_line":"            #     but not yet cleaved; and"},{"line_number":2353,"context_line":"            #  3. So we have an rough idea the size of the shards should be"},{"line_number":2354,"context_line":"            #     while cleaving."},{"line_number":2355,"context_line":"            found_ranges.append("},{"line_number":2356,"context_line":"                {\u0027index\u0027: index,"}],"source_content_type":"text/x-python","patch_set":2,"id":"f0460496_25797e71","line":2353,"range":{"start_line":2353,"start_character":29,"end_line":2353,"end_character":31},"updated":"2022-08-03 05:03:05.000000000","message":"nit: \"a\", and s/the size of/of the size/","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":2350,"context_line":"            #     constant;"},{"line_number":2351,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2352,"context_line":"            #     but not yet cleaved; and"},{"line_number":2353,"context_line":"            #  3. So we have an rough idea the size of the shards should be"},{"line_number":2354,"context_line":"            #     while cleaving."},{"line_number":2355,"context_line":"            found_ranges.append("},{"line_number":2356,"context_line":"                {\u0027index\u0027: index,"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa08089b_17d56663","line":2353,"range":{"start_line":2353,"start_character":29,"end_line":2353,"end_character":31},"in_reply_to":"f0460496_25797e71","updated":"2022-12-06 03:59:47.000000000","message":"Ack","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":1907,"context_line":"                SHARD_UPDATE_STAT_STATES:"},{"line_number":1908,"context_line":"            info \u003d self.get_info()"},{"line_number":1909,"context_line":"            own_shard_range.update_meta("},{"line_number":1910,"context_line":"                info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":1911,"context_line":"        return own_shard_range"},{"line_number":1912,"context_line":""},{"line_number":1913,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":4,"id":"18d4dfdf_ffce0655","line":1910,"updated":"2022-08-05 17:52:43.000000000","message":"I\u0027ve never really liked the way the stats are updated as a side effect (but not persisted) and it feels like this is getting more convoluted i.e. the side effect is now conditional, and depends on what feels more like sharding state than DB state. I wonder if the time has come to separate out the stats updating? \n\nI took a crack at it here https://review.opendev.org/c/openstack/swift/+/852283","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":1907,"context_line":"                SHARD_UPDATE_STAT_STATES:"},{"line_number":1908,"context_line":"            info \u003d self.get_info()"},{"line_number":1909,"context_line":"            own_shard_range.update_meta("},{"line_number":1910,"context_line":"                info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":1911,"context_line":"        return own_shard_range"},{"line_number":1912,"context_line":""},{"line_number":1913,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":4,"id":"ecd6e70b_3e3e7af8","line":1910,"in_reply_to":"18d4dfdf_ffce0655","updated":"2022-12-06 03:59:47.000000000","message":"Done","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":2352,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2353,"context_line":"            #     but not yet cleaved; and"},{"line_number":2354,"context_line":"            #  3. So we have a rough idea of size of the shards should be"},{"line_number":2355,"context_line":"            #     while cleaving."},{"line_number":2356,"context_line":"            found_ranges.append("},{"line_number":2357,"context_line":"                {\u0027index\u0027: index,"},{"line_number":2358,"context_line":"                 \u0027lower\u0027: str(last_shard_upper),"}],"source_content_type":"text/x-python","patch_set":4,"id":"6253eba2_0653207b","line":2355,"updated":"2022-08-05 17:52:43.000000000","message":"I like additional commentary to help us remember the *whys* of sharder behaviour! :)","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":false,"context_lines":[{"line_number":2352,"context_line":"            #  2. the container is non-deletable while shards have been found"},{"line_number":2353,"context_line":"            #     but not yet cleaved; and"},{"line_number":2354,"context_line":"            #  3. So we have a rough idea of size of the shards should be"},{"line_number":2355,"context_line":"            #     while cleaving."},{"line_number":2356,"context_line":"            found_ranges.append("},{"line_number":2357,"context_line":"                {\u0027index\u0027: index,"},{"line_number":2358,"context_line":"                 \u0027lower\u0027: str(last_shard_upper),"}],"source_content_type":"text/x-python","patch_set":4,"id":"60d0bb30_cdd7b85d","line":2355,"in_reply_to":"6253eba2_0653207b","updated":"2022-12-09 11:49:28.000000000","message":"Done","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":61,"context_line":"                         ShardRange.ACTIVE, ShardRange.SHARDING,"},{"line_number":62,"context_line":"                         ShardRange.SHARDED, ShardRange.SHRINKING,"},{"line_number":63,"context_line":"                         ShardRange.SHRUNK]"},{"line_number":64,"context_line":"SHARD_UPDATE_STAT_STATES \u003d [ShardRange.CLEAVED, ShardRange.ACTIVE,"},{"line_number":65,"context_line":"                            ShardRange.SHARDING, ShardRange.SHARDED,"},{"line_number":66,"context_line":"                            ShardRange.SHRINKING, ShardRange.SHRUNK]"},{"line_number":67,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"a0a3cbf6_2a285e44","line":64,"updated":"2022-12-09 11:49:28.000000000","message":"maybe add a comment\n\n  # shard\u0027s may not be fully populated while in the FOUND and CREATED\n  # state, so shards should only update their own shard range\u0027s object \n  # stats when they are in the following states","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":61,"context_line":"                         ShardRange.ACTIVE, ShardRange.SHARDING,"},{"line_number":62,"context_line":"                         ShardRange.SHARDED, ShardRange.SHRINKING,"},{"line_number":63,"context_line":"                         ShardRange.SHRUNK]"},{"line_number":64,"context_line":"SHARD_UPDATE_STAT_STATES \u003d [ShardRange.CLEAVED, ShardRange.ACTIVE,"},{"line_number":65,"context_line":"                            ShardRange.SHARDING, ShardRange.SHARDED,"},{"line_number":66,"context_line":"                            ShardRange.SHRINKING, ShardRange.SHRUNK]"},{"line_number":67,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"415e0c58_0c69195d","line":64,"in_reply_to":"a0a3cbf6_2a285e44","updated":"2022-12-12 06:42:13.000000000","message":"great idea!","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"48cd21075b20ba47c8674dd907dc9a074c9979ec","unresolved":true,"context_lines":[{"line_number":332,"context_line":"        range."},{"line_number":333,"context_line":"    :param existing_shard_ranges: a dict mapping shard range names to dicts"},{"line_number":334,"context_line":"        representing a shard range."},{"line_number":335,"context_line":"    :return: a tuple (to_add, to_delete); to_add is a list of dicts, each of"},{"line_number":336,"context_line":"        which represents a shard range that is to be added to the existing"},{"line_number":337,"context_line":"        shard ranges; to_delete is a set of shard range names that are to be"},{"line_number":338,"context_line":"        deleted."}],"source_content_type":"text/x-python","patch_set":13,"id":"879eeb1a_b3213316","line":335,"range":{"start_line":335,"start_character":42,"end_line":335,"end_character":67},"updated":"2023-02-01 22:48:50.000000000","message":"Well, an iterator of dicts, anyway -- I think I\u0027d be a fan of pulling them into a list later, though.","commit_id":"ece4b04e8215f80b55c367abc523290b503efc76"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":true,"context_lines":[{"line_number":2259,"context_line":"            self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2260,"context_line":""},{"line_number":2261,"context_line":"            # Now that we\u0027ve updated stats in the OSR, let\u0027s save them."},{"line_number":2262,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2263,"context_line":""},{"line_number":2264,"context_line":"        if own_shard_range.reported:"},{"line_number":2265,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":10,"id":"68a9ae61_23448a1b","line":2262,"updated":"2022-12-06 03:59:47.000000000","message":"I guess moving this into here changes what the old patch did. Previously the stats wouldn\u0027t change in the DB unless the range had been reported. Now it\u0027s changing on every cycle but not actually always updating the root.\n\nMaybe I should pull 2262 back out and down to line 2261. \nBut by moving it into the if-block means it\u0027ll only merge and write it out when something had changed (updated stats, relcaim, etc). :shrug:","commit_id":"c09844bd682a3e6ab05c917703ea532bfc393ac9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dbac0b989a6bfafe0dd616b3e5f5095f96f0db99","unresolved":true,"context_lines":[{"line_number":2259,"context_line":"            self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2260,"context_line":""},{"line_number":2261,"context_line":"            # Now that we\u0027ve updated stats in the OSR, let\u0027s save them."},{"line_number":2262,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2263,"context_line":""},{"line_number":2264,"context_line":"        if own_shard_range.reported:"},{"line_number":2265,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":10,"id":"a847de75_d7b2adf0","line":2262,"in_reply_to":"68a9ae61_23448a1b","updated":"2022-12-08 18:07:10.000000000","message":"I don\u0027t understand this statement \"Now it\u0027s changing on every cycle but not actually always updating the root.\" If the own_sr stats are changed by the call to update_own_shard_range_stats then won\u0027t own_shard_range.reported be False at line 2277?","commit_id":"c09844bd682a3e6ab05c917703ea532bfc393ac9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dbb8e6d8a92a43b1bfeb705ed995c8260aeb646b","unresolved":false,"context_lines":[{"line_number":2259,"context_line":"            self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2260,"context_line":""},{"line_number":2261,"context_line":"            # Now that we\u0027ve updated stats in the OSR, let\u0027s save them."},{"line_number":2262,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2263,"context_line":""},{"line_number":2264,"context_line":"        if own_shard_range.reported:"},{"line_number":2265,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":10,"id":"28177ec7_f246ca16","line":2262,"in_reply_to":"a847de75_d7b2adf0","updated":"2022-12-09 02:34:00.000000000","message":"yup your right. if the stats do get updated the reported is turned to False.","commit_id":"c09844bd682a3e6ab05c917703ea532bfc393ac9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dbac0b989a6bfafe0dd616b3e5f5095f96f0db99","unresolved":true,"context_lines":[{"line_number":2261,"context_line":"            return"},{"line_number":2262,"context_line":""},{"line_number":2263,"context_line":"        # Don\u0027t update the osr stats including tombstones unless its CLEAVED+"},{"line_number":2264,"context_line":"        if own_shard_range.state in SHARD_UPDATE_STAT_STATES:"},{"line_number":2265,"context_line":"            # do a reclaim *now* in order to get best estimate of tombstone"},{"line_number":2266,"context_line":"            # count that is consistent with the current object_count"},{"line_number":2267,"context_line":"            reclaimer \u003d self._reclaim(broker)"}],"source_content_type":"text/x-python","patch_set":11,"id":"c18e5f50_b30d7ed7","line":2264,"updated":"2022-12-08 18:07:10.000000000","message":"IDK, might it be ok to just list FOUND and CREATED?","commit_id":"70e57f8059c28b43934323bde1f44f8685ddda11"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dbb8e6d8a92a43b1bfeb705ed995c8260aeb646b","unresolved":true,"context_lines":[{"line_number":2261,"context_line":"            return"},{"line_number":2262,"context_line":""},{"line_number":2263,"context_line":"        # Don\u0027t update the osr stats including tombstones unless its CLEAVED+"},{"line_number":2264,"context_line":"        if own_shard_range.state in SHARD_UPDATE_STAT_STATES:"},{"line_number":2265,"context_line":"            # do a reclaim *now* in order to get best estimate of tombstone"},{"line_number":2266,"context_line":"            # count that is consistent with the current object_count"},{"line_number":2267,"context_line":"            reclaimer \u003d self._reclaim(broker)"}],"source_content_type":"text/x-python","patch_set":11,"id":"fdbf66e6_ddc14fdb","line":2264,"in_reply_to":"c18e5f50_b30d7ed7","updated":"2022-12-09 02:34:00.000000000","message":"oh you hean if it\u0027s not in FOUND and CREATED because the the otherway of saying it.. yeah that\u0027s true. But as it currently is more readable and hopefully maintainable.\n\nThough happy to be convinced the otherway.","commit_id":"70e57f8059c28b43934323bde1f44f8685ddda11"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":false,"context_lines":[{"line_number":2261,"context_line":"            return"},{"line_number":2262,"context_line":""},{"line_number":2263,"context_line":"        # Don\u0027t update the osr stats including tombstones unless its CLEAVED+"},{"line_number":2264,"context_line":"        if own_shard_range.state in SHARD_UPDATE_STAT_STATES:"},{"line_number":2265,"context_line":"            # do a reclaim *now* in order to get best estimate of tombstone"},{"line_number":2266,"context_line":"            # count that is consistent with the current object_count"},{"line_number":2267,"context_line":"            reclaimer \u003d self._reclaim(broker)"}],"source_content_type":"text/x-python","patch_set":11,"id":"8e4bd2ce_88bb39c4","line":2264,"in_reply_to":"fdbf66e6_ddc14fdb","updated":"2022-12-09 11:49:28.000000000","message":"it works either way, no big deal","commit_id":"70e57f8059c28b43934323bde1f44f8685ddda11"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":2268,"context_line":"            tombstones \u003d reclaimer.get_tombstone_count()"},{"line_number":2269,"context_line":"            self.logger.debug(\u0027tombstones in %s \u003d %d\u0027,"},{"line_number":2270,"context_line":"                              quote(broker.path), tombstones)"},{"line_number":2271,"context_line":"            own_shard_range.update_tombstones(tombstones)"},{"line_number":2272,"context_line":"            update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2273,"context_line":""},{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"}],"source_content_type":"text/x-python","patch_set":12,"id":"98aa2930_86616a1d","line":2271,"updated":"2022-12-09 11:49:28.000000000","message":"slightly off topic: I was wondering why we don\u0027t just roll the tombstone stats update into update_own_shard_range_stats, but remembered that this is the only place we care about up to date tombstone count i.e. when we\u0027re about to send stats to the root - might be worth adding a comment to that effect.\n\n(shrinking candidates are found in the root DB so that\u0027s the only place we need up to date tombstone stats)","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":2268,"context_line":"            tombstones \u003d reclaimer.get_tombstone_count()"},{"line_number":2269,"context_line":"            self.logger.debug(\u0027tombstones in %s \u003d %d\u0027,"},{"line_number":2270,"context_line":"                              quote(broker.path), tombstones)"},{"line_number":2271,"context_line":"            own_shard_range.update_tombstones(tombstones)"},{"line_number":2272,"context_line":"            update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2273,"context_line":""},{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"}],"source_content_type":"text/x-python","patch_set":12,"id":"1bbfcdab_be947207","line":2271,"in_reply_to":"98aa2930_86616a1d","updated":"2022-12-12 06:42:13.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":2272,"context_line":"            update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2273,"context_line":""},{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"},{"line_number":2275,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2276,"context_line":""},{"line_number":2277,"context_line":"        if own_shard_range.reported:"},{"line_number":2278,"context_line":"            # no change to the stats metadata"}],"source_content_type":"text/x-python","patch_set":12,"id":"81b68b77_ce587c88","line":2275,"updated":"2022-12-09 11:49:28.000000000","message":"this can be conditional:\n\n  if not own_shard_range.reported:\n      broker.merge_shard_ranges(own_shard_range)","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":2272,"context_line":"            update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2273,"context_line":""},{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"},{"line_number":2275,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2276,"context_line":""},{"line_number":2277,"context_line":"        if own_shard_range.reported:"},{"line_number":2278,"context_line":"            # no change to the stats metadata"}],"source_content_type":"text/x-python","patch_set":12,"id":"d6944a4c_accaec18","line":2275,"in_reply_to":"81b68b77_ce587c88","updated":"2022-12-12 06:42:13.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"},{"line_number":2275,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2276,"context_line":""},{"line_number":2277,"context_line":"        if own_shard_range.reported:"},{"line_number":2278,"context_line":"            # no change to the stats metadata"},{"line_number":2279,"context_line":"            return"},{"line_number":2280,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"16ac1be2_34131437","line":2277,"updated":"2022-12-09 11:49:28.000000000","message":"note to self: we cannot simply write, earlier,\n\n  if own_shard_range.state not in SHARD_UPDATE_STAT_STATES:\n      return\n      \nbecause:\n\n1. this may be the first pass over a new shard DB so the initial own shard range has not been reported to root before: we do still want to let the root know about new shard ranges, even if we don\u0027t want them to report their dynamic stats yet.\n\n2. hypothetical: the shard was in SHARD_UPDATE_STAT_STATES, updated its stats, but the PUT to root failed. The shard is now not in SHARD_UPDATE_STAT_STATES but the latest stats have not been reported. Hypothetical because we do not expect shard state to revert from being in SHARD_UPDATE_STAT_STATES to not being in SHARD_UPDATE_STAT_STATES.","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b2cb3604b651eca69a0e926d9005aa5509b4d33d","unresolved":false,"context_lines":[{"line_number":2274,"context_line":"            # stats metadata has been updated so persist it"},{"line_number":2275,"context_line":"            broker.merge_shard_ranges(own_shard_range)"},{"line_number":2276,"context_line":""},{"line_number":2277,"context_line":"        if own_shard_range.reported:"},{"line_number":2278,"context_line":"            # no change to the stats metadata"},{"line_number":2279,"context_line":"            return"},{"line_number":2280,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"5f7a2070_0b7517d0","line":2277,"in_reply_to":"16ac1be2_34131437","updated":"2022-12-12 06:42:13.000000000","message":"yeah, I think its just safer to use the .reported as latching is its job anyway.","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"}],"test/unit/container/test_backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ccd0f2672102dbf45c5399e43fa70be7fdc0c1a","unresolved":true,"context_lines":[{"line_number":4353,"context_line":"                            return_value\u003dnext_ts):"},{"line_number":4354,"context_line":"                actual \u003d broker.get_own_shard_range()"},{"line_number":4355,"context_line":""},{"line_number":4356,"context_line":"            if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":4357,"context_line":"                expected \u003d dict(ShardRange("},{"line_number":4358,"context_line":"                    broker.path, now, \u0027l\u0027, \u0027u\u0027, num_objects_in_broker[0],"},{"line_number":4359,"context_line":"                    num_objects_in_broker[0] * 100,"}],"source_content_type":"text/x-python","patch_set":2,"id":"0db22b2a_50f651b5","line":4356,"updated":"2022-08-03 05:03:05.000000000","message":"The nature of this test doesn\u0027t make it obvious which states get the exclusion and which don\u0027t...","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a5f46d9491d1a08048f840a0eacc0f2703f1a3a6","unresolved":false,"context_lines":[{"line_number":4353,"context_line":"                            return_value\u003dnext_ts):"},{"line_number":4354,"context_line":"                actual \u003d broker.get_own_shard_range()"},{"line_number":4355,"context_line":""},{"line_number":4356,"context_line":"            if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":4357,"context_line":"                expected \u003d dict(ShardRange("},{"line_number":4358,"context_line":"                    broker.path, now, \u0027l\u0027, \u0027u\u0027, num_objects_in_broker[0],"},{"line_number":4359,"context_line":"                    num_objects_in_broker[0] * 100,"}],"source_content_type":"text/x-python","patch_set":2,"id":"36745f3a_8987537a","line":4356,"in_reply_to":"0db22b2a_50f651b5","updated":"2022-08-03 07:37:57.000000000","message":"yup, good point, I\u0027ll specifically use them rather the group.","commit_id":"a0ce85a8c4a84a4c7cff4b461e3864837cdf81ed"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"49f119b2531f516439a3cd0523adb66213cef928","unresolved":true,"context_lines":[{"line_number":5004,"context_line":"                    expected_sent \u003d ["},{"line_number":5005,"context_line":"                        dict(own_shard_range,"},{"line_number":5006,"context_line":"                             meta_timestamp\u003dnow.internal,"},{"line_number":5007,"context_line":"                             tombstones\u003down_shard_range.tombstones + 1)]"},{"line_number":5008,"context_line":"                self.check_shard_ranges_sent(broker, expected_sent)"},{"line_number":5009,"context_line":""},{"line_number":5010,"context_line":"        for state in ShardRange.STATES:"}],"source_content_type":"text/x-python","patch_set":1,"id":"b553b7d6_2c6a227d","line":5007,"range":{"start_line":5007,"start_character":29,"end_line":5007,"end_character":70},"updated":"2022-04-14 02:07:39.000000000","message":"atm tombstone updates do come through.. but maybe this isn\u0027t a bad thing. I wonder if we can use it to help determine a better estimated expected cleave size?","commit_id":"1be971df020fb769d065747b92d268327241c5c8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":5004,"context_line":"                    expected_sent \u003d ["},{"line_number":5005,"context_line":"                        dict(own_shard_range,"},{"line_number":5006,"context_line":"                             meta_timestamp\u003dnow.internal,"},{"line_number":5007,"context_line":"                             tombstones\u003down_shard_range.tombstones + 1)]"},{"line_number":5008,"context_line":"                self.check_shard_ranges_sent(broker, expected_sent)"},{"line_number":5009,"context_line":""},{"line_number":5010,"context_line":"        for state in ShardRange.STATES:"}],"source_content_type":"text/x-python","patch_set":1,"id":"5d0a9ae8_6a0f1f16","line":5007,"range":{"start_line":5007,"start_character":29,"end_line":5007,"end_character":70},"in_reply_to":"b553b7d6_2c6a227d","updated":"2022-12-06 03:59:47.000000000","message":"Ack","commit_id":"1be971df020fb769d065747b92d268327241c5c8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":5075,"context_line":"                              next(self.ts_iter).internal, 1, \u0027\u0027, \u0027\u0027)"},{"line_number":5076,"context_line":"            with mock_timestamp_now() as now:"},{"line_number":5077,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"}],"source_content_type":"text/x-python","patch_set":4,"id":"d83f7f25_e5eeebe4","line":5078,"updated":"2022-08-05 17:52:43.000000000","message":"nit: could this be named exp_own_shard_range?","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fc3129618c1ba4c6f7e5eb7faf93b1c97a627c06","unresolved":false,"context_lines":[{"line_number":5075,"context_line":"                              next(self.ts_iter).internal, 1, \u0027\u0027, \u0027\u0027)"},{"line_number":5076,"context_line":"            with mock_timestamp_now() as now:"},{"line_number":5077,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"}],"source_content_type":"text/x-python","patch_set":4,"id":"9475265d_f1ed70f4","line":5078,"in_reply_to":"d83f7f25_e5eeebe4","updated":"2022-11-30 01:32:27.000000000","message":"Done","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":5076,"context_line":"            with mock_timestamp_now() as now:"},{"line_number":5077,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"}],"source_content_type":"text/x-python","patch_set":4,"id":"2780240f_2295b75d","line":5079,"updated":"2022-08-05 17:52:43.000000000","message":"this is odd: meta_timestamp is always set even when the stats are not updated? I think that is because we *do* still update the tombstones count, which is not consistent with not updating the object count. But I\u0027m also concerned that we\u0027re setting meta_timestamp to now but for a stale value of object_count.","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":5076,"context_line":"            with mock_timestamp_now() as now:"},{"line_number":5077,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"}],"source_content_type":"text/x-python","patch_set":4,"id":"63a36704_7b86b003","line":5079,"in_reply_to":"1b67137a_733fd156","updated":"2022-12-06 03:59:47.000000000","message":"Done","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"24fb50e2f5713328b08ba19ab0686470042fb7d8","unresolved":true,"context_lines":[{"line_number":5076,"context_line":"            with mock_timestamp_now() as now:"},{"line_number":5077,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"}],"source_content_type":"text/x-python","patch_set":4,"id":"1b67137a_733fd156","line":5079,"in_reply_to":"2780240f_2295b75d","updated":"2022-11-30 00:37:00.000000000","message":"hmm, yeah that\u0027s a good point. So maybe we to not update tombstone counts unless its\u0027 CLEAVED+ as well","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa191cee_ed4ae484","line":5081,"updated":"2022-08-05 17:52:43.000000000","message":"this is a no-op (meta_timestamp was set on line 5079)","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":5078,"context_line":"                tmp_own_shard_range \u003d own_shard_range.copy("},{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"}],"source_content_type":"text/x-python","patch_set":4,"id":"a40aadc4_06910872","line":5081,"in_reply_to":"fa191cee_ed4ae484","updated":"2022-12-06 03:59:47.000000000","message":"Ack","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"},{"line_number":5085,"context_line":"                        own_shard_range.bytes_used + 1"}],"source_content_type":"text/x-python","patch_set":4,"id":"c5553139_90978a48","line":5082,"updated":"2022-08-05 17:52:43.000000000","message":"can be\n\n  tmp_own_shard_range.object_count +\u003d 1","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":5079,"context_line":"                    meta_timestamp\u003dnow.internal, tombstones\u003d0)"},{"line_number":5080,"context_line":"                if state in SHARD_UPDATE_STAT_STATES:"},{"line_number":5081,"context_line":"                    tmp_own_shard_range.meta_timestamp \u003d now.internal"},{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"},{"line_number":5085,"context_line":"                        own_shard_range.bytes_used + 1"}],"source_content_type":"text/x-python","patch_set":4,"id":"a0eafc22_fb80e4bc","line":5082,"in_reply_to":"c5553139_90978a48","updated":"2022-12-06 03:59:47.000000000","message":"Ack","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98acb69c5ee4b8e8b3c137f6177c5b179cf77fb8","unresolved":true,"context_lines":[{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"},{"line_number":5085,"context_line":"                        own_shard_range.bytes_used + 1"},{"line_number":5086,"context_line":"                expected_sent \u003d sorted(["},{"line_number":5087,"context_line":"                    tmp_own_shard_range] + shard_ranges,"},{"line_number":5088,"context_line":"                    key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"}],"source_content_type":"text/x-python","patch_set":4,"id":"9cc82d23_6dc6c4e3","line":5085,"updated":"2022-08-05 17:52:43.000000000","message":"can be\n  tmp_own_shard_range.bytes_used +\u003d 1","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5505c401cda3fc236f8e71b37f69e4fc8dbc88a6","unresolved":false,"context_lines":[{"line_number":5082,"context_line":"                    tmp_own_shard_range.object_count \u003d \\"},{"line_number":5083,"context_line":"                        own_shard_range.object_count + 1"},{"line_number":5084,"context_line":"                    tmp_own_shard_range.bytes_used \u003d \\"},{"line_number":5085,"context_line":"                        own_shard_range.bytes_used + 1"},{"line_number":5086,"context_line":"                expected_sent \u003d sorted(["},{"line_number":5087,"context_line":"                    tmp_own_shard_range] + shard_ranges,"},{"line_number":5088,"context_line":"                    key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"}],"source_content_type":"text/x-python","patch_set":4,"id":"e8063699_cd96fd5e","line":5085,"in_reply_to":"9cc82d23_6dc6c4e3","updated":"2022-12-06 03:59:47.000000000","message":"Ack","commit_id":"c4d615b64b3b07126f7d25f6545e253aa89a2e09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":5314,"context_line":"                             bytes_used\u003dlen(obj_names))]"},{"line_number":5315,"context_line":"                else:"},{"line_number":5316,"context_line":"                    expected_sent \u003d ["},{"line_number":5317,"context_line":"                        dict(own_shard_range)]"},{"line_number":5318,"context_line":"                self.check_shard_ranges_sent(broker, expected_sent)"},{"line_number":5319,"context_line":""},{"line_number":5320,"context_line":"        # initialise tombstones"}],"source_content_type":"text/x-python","patch_set":12,"id":"58fc4263_857494d8","line":5317,"updated":"2022-12-09 11:49:28.000000000","message":"we should also check that the broker was (or was not) updated with the stats (see diff pasted below)","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e5d34a4e957a84c826909384790baed8ad179855","unresolved":false,"context_lines":[{"line_number":5314,"context_line":"                             bytes_used\u003dlen(obj_names))]"},{"line_number":5315,"context_line":"                else:"},{"line_number":5316,"context_line":"                    expected_sent \u003d ["},{"line_number":5317,"context_line":"                        dict(own_shard_range)]"},{"line_number":5318,"context_line":"                self.check_shard_ranges_sent(broker, expected_sent)"},{"line_number":5319,"context_line":""},{"line_number":5320,"context_line":"        # initialise tombstones"}],"source_content_type":"text/x-python","patch_set":12,"id":"df3fe907_fed424e6","line":5317,"in_reply_to":"58fc4263_857494d8","updated":"2022-12-13 12:45:39.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"02569842f310b91c41d2288b9cd78d14ba3db2e8","unresolved":true,"context_lines":[{"line_number":5358,"context_line":"                    exp_tombstones \u003d i + 1"},{"line_number":5359,"context_line":"                else:"},{"line_number":5360,"context_line":"                    exp_tombstones \u003d 0"},{"line_number":5361,"context_line":"                check_tombstones_sent(state, exp_tombstones)"},{"line_number":5362,"context_line":""},{"line_number":5363,"context_line":"    def test_update_root_container_already_reported(self):"},{"line_number":5364,"context_line":"        broker \u003d self._make_broker()"}],"source_content_type":"text/x-python","patch_set":12,"id":"684cd07f_14114fb2","line":5361,"updated":"2022-12-09 11:49:28.000000000","message":"diff to make this a little simpler and more consistent with check_only_own_shard_range_sent\n\nhttps://paste.openstack.org/show/bBRufPncuA6GuciceVMV/","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e5d34a4e957a84c826909384790baed8ad179855","unresolved":false,"context_lines":[{"line_number":5358,"context_line":"                    exp_tombstones \u003d i + 1"},{"line_number":5359,"context_line":"                else:"},{"line_number":5360,"context_line":"                    exp_tombstones \u003d 0"},{"line_number":5361,"context_line":"                check_tombstones_sent(state, exp_tombstones)"},{"line_number":5362,"context_line":""},{"line_number":5363,"context_line":"    def test_update_root_container_already_reported(self):"},{"line_number":5364,"context_line":"        broker \u003d self._make_broker()"}],"source_content_type":"text/x-python","patch_set":12,"id":"e93944d2_05be0a27","line":5361,"in_reply_to":"684cd07f_14114fb2","updated":"2022-12-13 12:45:39.000000000","message":"Done","commit_id":"1a47dd863c3857803779b4f030bb6708f511e766"}]}