)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2e29fba43155b45f0abb305b0cb789adc3ba7b78","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"14435ba4_4794b29e","updated":"2022-07-07 15:26:51.000000000","message":"IDK, I think looking at the databases in prod we just sort of decided \"these aren\u0027t really deleted yet\"... but maybe it all just depends on how you define \"is_deleted\"","commit_id":"15518e7138b6ed04ac60f29ac4544bfd20f402e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e311cc50897a4a08fa6423af3780a684c6455100","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"cfcc25da_a37bd6aa","updated":"2022-07-07 17:52:59.000000000","message":"let\u0027s try to make is_deleted better and keep this patch in reserve, but the tests herein are useful for any solution","commit_id":"bf0691ce5fd1dd7a4978a06fb8ae6f527f6cd45e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b777c54b4cb10ce422e24e319908016ac2133370","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"a2fd7479_5c6ae53b","updated":"2022-07-08 16:18:07.000000000","message":"I changed direction in this patchset to modify the broker is_deleted algorithm","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6532ab70e552ebed0b932c5bdcfe68cc3e8eab2b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"61b45088_a2eb2fa8","updated":"2022-07-08 17:42:31.000000000","message":"I think this is great work Al, I know Matt has been thinking about this a lot too.  I\u0027m sure y\u0027all will figure out what to do.  Alohda!","commit_id":"4914033e34ff7e857e595d0308aac8763885d511"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4b78fcac13179a3c2bac9054644294f448585b5a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"7976eef5_e2493f95","updated":"2022-07-08 17:33:41.000000000","message":"one of the failing tests reveals something to ponder - when synthesizing a filler range to fill a gap in shard ranges, we use a copy of the own_sr, which for an unsharded db with no objects will now have object_count\u003d\u003dget_shard_usage() whereas before it was zero. zero seems a better choice for a synthesized filler range.\n\n#willfix","commit_id":"4914033e34ff7e857e595d0308aac8763885d511"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f402e36d409bf3dd8f5b33bf1454220b4a93bb40","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"46b539a6_c08e9a63","updated":"2022-07-11 18:17:31.000000000","message":"@Matt - tweaked so that we don\u0027t report object count when there is no object to list\n\n","commit_id":"46e2920b9b68965d8842b302aa6fbb40da930cb6"}],"swift/container/backend.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6532ab70e552ebed0b932c5bdcfe68cc3e8eab2b","unresolved":true,"context_lines":[{"line_number":820,"context_line":"            SELECT put_timestamp, delete_timestamp, object_count"},{"line_number":821,"context_line":"            FROM container_stat\u0027\u0027\u0027).fetchone()"},{"line_number":822,"context_line":"        info \u003d dict(info)"},{"line_number":823,"context_line":"        info.update(self._get_alternate_object_stats()[1])"},{"line_number":824,"context_line":"        return self._is_deleted_info(**info)"},{"line_number":825,"context_line":""},{"line_number":826,"context_line":"    def is_old_enough_to_reclaim(self, now, reclaim_age):"}],"source_content_type":"text/x-python","patch_set":4,"id":"68f4bc76_f336d9ba","side":"PARENT","line":823,"updated":"2022-07-08 17:42:31.000000000","message":"i\u0027m glad to see the subtle return unpacking gone here","commit_id":"6de2880af61997b1d5c389e65d059e4a6a015c06"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6532ab70e552ebed0b932c5bdcfe68cc3e8eab2b","unresolved":true,"context_lines":[{"line_number":933,"context_line":"        data \u003d self._get_info()"},{"line_number":934,"context_line":"        state, stats \u003d self._get_alternate_object_stats()"},{"line_number":935,"context_line":"        data.update(stats)"},{"line_number":936,"context_line":"        data[\u0027db_state\u0027] \u003d state"},{"line_number":937,"context_line":"        return data"},{"line_number":938,"context_line":""},{"line_number":939,"context_line":"    def set_x_container_sync_points(self, sync_point1, sync_point2):"}],"source_content_type":"text/x-python","patch_set":4,"id":"ef2cf6a5_4716c629","side":"PARENT","line":936,"updated":"2022-07-08 17:42:31.000000000","message":"oic, this is the only method that \"needed\" state returned from alternate_stats","commit_id":"6de2880af61997b1d5c389e65d059e4a6a015c06"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"29e7c75719c8d694ae2a4902b1d181c1c596eec2","unresolved":true,"context_lines":[{"line_number":910,"context_line":"        # update DB info dict stats items from alternative sources and add"},{"line_number":911,"context_line":"        # db_state to the info dict"},{"line_number":912,"context_line":"        state \u003d self.get_db_state()"},{"line_number":913,"context_line":"        info[\u0027db_state\u0027] \u003d state"},{"line_number":914,"context_line":""},{"line_number":915,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":916,"context_line":"            # while in DB is SHARDING use stats from the retiring DB"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa9aa58e_069e3840","line":913,"updated":"2022-07-08 17:05:01.000000000","message":"it turns out that the caller that did not previously need this key in the returned dict also does not care if is present...so we can just add it always","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"29e7c75719c8d694ae2a4902b1d181c1c596eec2","unresolved":true,"context_lines":[{"line_number":918,"context_line":"            info[\u0027object_count\u0027] \u003d alternate_stats[\u0027object_count\u0027]"},{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"}],"source_content_type":"text/x-python","patch_set":4,"id":"5172ecc1_92231ef3","line":921,"range":{"start_line":921,"start_character":13,"end_line":921,"end_character":58},"updated":"2022-07-08 17:05:01.000000000","message":"IIUC we definitely do not want shard containers to report their shard range stats once they have SHARDED (which could be due to sharding or shrinking)","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9d92c46d1bb6b5d652fb2ad31da90911773cf188","unresolved":true,"context_lines":[{"line_number":918,"context_line":"            info[\u0027object_count\u0027] \u003d alternate_stats[\u0027object_count\u0027]"},{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"}],"source_content_type":"text/x-python","patch_set":4,"id":"600750a4_f431383a","line":921,"range":{"start_line":921,"start_character":13,"end_line":921,"end_character":58},"in_reply_to":"5172ecc1_92231ef3","updated":"2022-07-11 07:08:26.000000000","message":"Oh yeah, good call","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"29e7c75719c8d694ae2a4902b1d181c1c596eec2","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"},{"line_number":926,"context_line":"            # SHARDING db\u0027s to appear empty when the shard ranges do report"}],"source_content_type":"text/x-python","patch_set":4,"id":"36a7b58b_17b4595d","line":923,"range":{"start_line":922,"start_character":16,"end_line":923,"end_character":50},"updated":"2022-07-08 17:05:01.000000000","message":"apart from the refactoring, this is the only behavioural change","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"70eaa5e5456a16e354a0d992880fbdaf2ae1f538","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"},{"line_number":926,"context_line":"            # SHARDING db\u0027s to appear empty when the shard ranges do report"}],"source_content_type":"text/x-python","patch_set":4,"id":"c4c7622b_0619727c","line":923,"range":{"start_line":922,"start_character":16,"end_line":923,"end_character":50},"in_reply_to":"2273aa7c_350fb92a","updated":"2022-07-11 09:12:39.000000000","message":"@Clay: in the past we decided that when a db is UNSHARDED or SHARDING the most sensible indication of object count was the (retiring) DB: while cleaving, objects are not removed from the retiring DB so it continues to report its historic object_count; there\u0027s no sure way of knowing that the shards object counts are more accurate than the retiring DB at this point[*]. Once sharded we flip to use the shards\u0027 object counts, knowing that all of the db\u0027s rows have been cleaved so the shards\u0027 object counts will be at least the retiring db object count - but we only do this for root containers - sharded shards should always be considered empty.\n\nI am preserving that behaviour except for when the retiring DB object count is zero (when we see the bug) - in this case if the shard ranges have non-zero object count then this *is* better info than we have from the retiring DB, so we\u0027ll now use that.\n\nA COLLAPSED db by definition has no shard ranges.\n\n([*] on reflection, it could be argued that we should always use the greater of retiring db object count or shards\u0027 object count.)","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6532ab70e552ebed0b932c5bdcfe68cc3e8eab2b","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"},{"line_number":926,"context_line":"            # SHARDING db\u0027s to appear empty when the shard ranges do report"}],"source_content_type":"text/x-python","patch_set":4,"id":"e3ce87ec_7558b2ab","line":923,"range":{"start_line":922,"start_character":16,"end_line":923,"end_character":50},"in_reply_to":"36a7b58b_17b4595d","updated":"2022-07-08 17:42:31.000000000","message":"why the state condition and not object_count?\n\nNOTFOUND \u003d \u0027not_found\u0027\nUNSHARDED \u003d \u0027unsharded\u0027\nSHARDING \u003d \u0027sharding\u0027\nSHARDED \u003d \u0027sharded\u0027\nCOLLAPSED \u003d \u0027collapsed\u0027\n\n... did the tests about \"object_count should be zero\" have something to do with these:  https://review.opendev.org/c/openstack/swift/+/848776/1\n\n... I think there\u0027s *obviously* cases where we\u0027re returning object_count\u003d0 today and that\u0027s just WRONG - at a minimum, for the purposes of \"is_deleted\", I can\u0027t see a good justification that it\u0027s MOST correct to ever say yes if there\u0027s object count in either misplaced objects OR in the shard ranges table.  I\u0027m not sure how much we should let the complexity of the sharding state machine trample on that basic fact.","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"53fe4eada8d251f2f832892557fe6d66d3a321a2","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"},{"line_number":926,"context_line":"            # SHARDING db\u0027s to appear empty when the shard ranges do report"}],"source_content_type":"text/x-python","patch_set":4,"id":"2273aa7c_350fb92a","line":923,"range":{"start_line":922,"start_character":16,"end_line":923,"end_character":50},"in_reply_to":"70b7a3b8_66930d13","updated":"2022-07-11 06:45:49.000000000","message":"I guess the sharder does still try and deal with misplaced objects if the container is deleted, so it should \"undelete\" itself once it\u0027s dealt with any rows in the object table that are misplaced.. so maybe that part is ok. Not sure about the reconiler though.","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4de6dd23bb85920a1e7a625f49ad10e6947a3dbb","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"},{"line_number":926,"context_line":"            # SHARDING db\u0027s to appear empty when the shard ranges do report"}],"source_content_type":"text/x-python","patch_set":4,"id":"70b7a3b8_66930d13","line":923,"range":{"start_line":922,"start_character":16,"end_line":923,"end_character":50},"in_reply_to":"e3ce87ec_7558b2ab","updated":"2022-07-11 06:38:19.000000000","message":"From the time I spent on this last week I do now have a worry that our \"is_deleted\" is very flakey. The _is_deleted even mentions that is only is looking at the one spi. does this mean if there were objects in the wrong policy in the container, we can decide we\u0027re deleted. what happens to the reconciler, does this lead to dark data?\n\nFurther, what if we\u0027re sharded and our shards have no objects (great) but there are some misplaced objects in our object table.. then we have objects don\u0027t we?\n\nThe more I think about this, the more I want to make the _is_deleted simple. If there are objects in any policy in the object table (really that\u0027s just checking the policy stat table) or objects in the shards_ranges (for the states Al mentions) then we have objects and so can\u0027t be deleted.\nThis means we bail on thinking it\u0027s deleted and leave it to the consistency engine to fix it, be it sharding, reconciling or whatever.\n\nBUT I only want this on the _is_deleted check, not normal stats.. because as stats go, I think we do then need to looks for the right places when in different states.","commit_id":"7f9cd13d58b0ad6513a15f9e0bc61fa9222e47e6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9d92c46d1bb6b5d652fb2ad31da90911773cf188","unresolved":true,"context_lines":[{"line_number":820,"context_line":"            SELECT put_timestamp, delete_timestamp, object_count"},{"line_number":821,"context_line":"            FROM container_stat\u0027\u0027\u0027).fetchone()"},{"line_number":822,"context_line":"        info \u003d dict(info)"},{"line_number":823,"context_line":"        self._update_alternate_object_stats(info)"},{"line_number":824,"context_line":"        return self._is_deleted_info(**info)"},{"line_number":825,"context_line":""},{"line_number":826,"context_line":"    def is_old_enough_to_reclaim(self, now, reclaim_age):"}],"source_content_type":"text/x-python","patch_set":5,"id":"40efbe31_8f81e8b0","line":823,"updated":"2022-07-11 07:08:26.000000000","message":"Oh this looks much cleaner.","commit_id":"4914033e34ff7e857e595d0308aac8763885d511"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9d92c46d1bb6b5d652fb2ad31da90911773cf188","unresolved":true,"context_lines":[{"line_number":919,"context_line":"            info[\u0027bytes_used\u0027] \u003d alternate_stats[\u0027bytes_used\u0027]"},{"line_number":920,"context_line":""},{"line_number":921,"context_line":"        if ((state \u003d\u003d SHARDED and self.is_root_container())"},{"line_number":922,"context_line":"                or (state in (UNSHARDED, SHARDING)"},{"line_number":923,"context_line":"                    and not info[\u0027object_count\u0027])):"},{"line_number":924,"context_line":"            # use shard range stats if db is SHARDED or in other states if"},{"line_number":925,"context_line":"            # current object_count is zero; we do not want UNSHARDED or"}],"source_content_type":"text/x-python","patch_set":5,"id":"512efc58_a231a954","line":922,"range":{"start_line":922,"start_character":41,"end_line":922,"end_character":49},"updated":"2022-07-11 07:08:26.000000000","message":"So sharding appears twice. So to get here, this info check on the next line is actaully looking at the object_count of the retiring db. And only if that is 0.. I guess that\u0027s fine, in 99% of cases we\u0027re sharding a container.\n\nIf we\u0027re repairing a DB then it \"could\" be empty I guess, it\u0027ll be cleaving into a acceptor, which might not be able to happen if deleted. but this seems like such a niche case.. yeah I think this is fine","commit_id":"4914033e34ff7e857e595d0308aac8763885d511"}],"swift/container/sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2e29fba43155b45f0abb305b0cb789adc3ba7b78","unresolved":true,"context_lines":[{"line_number":2040,"context_line":"        self._move_misplaced_objects(broker)"},{"line_number":2041,"context_line":""},{"line_number":2042,"context_line":"        if (broker.is_deleted()"},{"line_number":2043,"context_line":"                and not broker.get_shard_usage()[\u0027object_count\u0027]):"},{"line_number":2044,"context_line":"            # This container is empty and deleted so we can skip it. Note that"},{"line_number":2045,"context_line":"            # it is possible for an unsharded container replica to appear"},{"line_number":2046,"context_line":"            # deleted (having no objects in the DB) but for the container to"}],"source_content_type":"text/x-python","patch_set":2,"id":"13b58b75_5d024c0e","line":2043,"updated":"2022-07-07 15:26:51.000000000","message":"I agree this change could be a solid option, and it might have been great to have a month ago so we could have drained those nodes already without having to get distracted by the in-memory database connection stuff - but IMHO it\u0027s the wrong fix and I wouldn\u0027t yet recommend merging it.\n\nI think matt and I had agreed: if a database has shard ranges with object count in them - it\u0027s not deleted (doesn\u0027t matter what timestamps say) - same goes for object rows both at the same time always!  You don\u0027t get to pick and choose when you follow the rules - the is_deleted function has a bug.  I\u0027m not sure yet what the consequences of fixing that bug will be...\n\nThere\u0027s maybe another question of if the sharder truely has nothing to do when a database is deleted or if the replicator truely has nothing to do if a database is sharded... but add additional conditionals seems like ADDING complexity.","commit_id":"15518e7138b6ed04ac60f29ac4544bfd20f402e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"69388dd0aa3c3d56732c6d30dcd3a28f6b8786fe","unresolved":true,"context_lines":[{"line_number":2040,"context_line":"        self._move_misplaced_objects(broker)"},{"line_number":2041,"context_line":""},{"line_number":2042,"context_line":"        if (broker.is_deleted()"},{"line_number":2043,"context_line":"                and not broker.get_shard_usage()[\u0027object_count\u0027]):"},{"line_number":2044,"context_line":"            # This container is empty and deleted so we can skip it. Note that"},{"line_number":2045,"context_line":"            # it is possible for an unsharded container replica to appear"},{"line_number":2046,"context_line":"            # deleted (having no objects in the DB) but for the container to"}],"source_content_type":"text/x-python","patch_set":2,"id":"6954bf0f_3fdcc47c","line":2043,"in_reply_to":"13b58b75_5d024c0e","updated":"2022-07-07 17:40:42.000000000","message":"The fundamental bug is that our sync engine (combination of replicator and sharder) is not bringing replicas into sync. The fix here solves that by un-stalling the sharder.\n\nHowever, I agree that all replicas already have sufficient state to make a consistent decision, and the ideal fix would be to make the replicas appear to be consistent sooner by making is_deleted() consistent across replicas. I just put this up because the other patches seem to be sprawling and this might be a useful tactical alternative.\n\nI\u0027ll keep digging into https://review.opendev.org/c/openstack/swift/+/848780 too.","commit_id":"15518e7138b6ed04ac60f29ac4544bfd20f402e4"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e311cc50897a4a08fa6423af3780a684c6455100","unresolved":true,"context_lines":[{"line_number":4813,"context_line":"        self.assertFalse(broker.logger.get_lines_for_level(\u0027warning\u0027))"},{"line_number":4814,"context_line":"        self.assertFalse(broker.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4815,"context_line":""},{"line_number":4816,"context_line":"    def test_process_broker_container_deleted_with_non_empty_shards(self):"},{"line_number":4817,"context_line":"        # verify that sharding process does not skip deleted containers if they"},{"line_number":4818,"context_line":"        # have shard ranges with non-zero objects"},{"line_number":4819,"context_line":"        broker \u003d self._make_broker(put_timestamp\u003dfloat(next(self.ts_iter)))"}],"source_content_type":"text/x-python","patch_set":3,"id":"fcb658e3_973f7a37","line":4816,"updated":"2022-07-07 17:52:59.000000000","message":"this test ought to be useful with whichever solution we choose","commit_id":"bf0691ce5fd1dd7a4978a06fb8ae6f527f6cd45e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6532ab70e552ebed0b932c5bdcfe68cc3e8eab2b","unresolved":true,"context_lines":[{"line_number":4813,"context_line":"        self.assertFalse(broker.logger.get_lines_for_level(\u0027warning\u0027))"},{"line_number":4814,"context_line":"        self.assertFalse(broker.logger.get_lines_for_level(\u0027error\u0027))"},{"line_number":4815,"context_line":""},{"line_number":4816,"context_line":"    def test_process_broker_container_deleted_with_non_empty_shards(self):"},{"line_number":4817,"context_line":"        # verify that sharding process does not skip deleted containers if they"},{"line_number":4818,"context_line":"        # have shard ranges with non-zero objects"},{"line_number":4819,"context_line":"        broker \u003d self._make_broker(put_timestamp\u003dfloat(next(self.ts_iter)))"}],"source_content_type":"text/x-python","patch_set":3,"id":"8883ff65_eac7e994","line":4816,"in_reply_to":"fcb658e3_973f7a37","updated":"2022-07-08 17:42:31.000000000","message":"you could move it all the way to the front of the change where you add test_process_broker_container_deleted except mark this one as expected failure https://docs.python.org/3/library/unittest.html#unittest.expectedFailure","commit_id":"bf0691ce5fd1dd7a4978a06fb8ae6f527f6cd45e"}]}