)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d5066e6a20ba50220248f14108b4a5ceeac95853","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"55d0b050_721244cd","updated":"2022-07-13 01:14:45.000000000","message":"Yup, this is good. We still process deleted cotainers, but once they get SHARDED there isn\u0027t really much work to do, so no need to have a special short cut for them anymore.\n\nI think this is also only possible now because we don\u0027t delete all the shard meta when we delete a DB. When the sharder was first written it was, so we couldn\u0027t easily process them, so we just dealt with misplaced objects and nothing else.\n\nJust a question in the probe test, not really a blocker though. so will -1 for the question, but think this is pretty much ready to land in my opinion. ","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"191d237ff7d2f4dc27959eb5d48132cbeb58404b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"23001057_51af3ba5","updated":"2022-07-12 17:29:29.000000000","message":"recheck\n\nunrelated failures","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05a7746c32495c9453d3e883b9144839e979bef3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"15da0e1c_133c3b90","updated":"2022-07-13 16:48:49.000000000","message":"added another probe test for a DB that appears deleted when other replicas have in fact cleaved objects to shards","commit_id":"51a076df7ae6ddd49a02ae8e022f2c6ceb1bc446"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2c7178b34e48de0a167b6b4c824ca3c54e47e28a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"dde7ec3f_d21a7272","updated":"2022-07-23 05:18:54.000000000","message":"I think I\u0027m on-board -- I\u0027m a little nervous about auto-sharding, though. I think if we had\n\n is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self.auto_shard and not broker.is_deleted()\n\nI\u0027d like it more; I think that should basically reduce the new branches we\u0027d hit to something closer to\n\n if broker.get_shard_ranges():\n     if broker.set_sharding_state():\n         state \u003d SHARDING\n if state \u003d\u003d SHARDING:\n     self._cleave(broker)","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d5066e6a20ba50220248f14108b4a5ceeac95853","unresolved":true,"context_lines":[{"line_number":1006,"context_line":"            self.logger.warning("},{"line_number":1007,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"},{"line_number":1008,"context_line":"                node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027],"},{"line_number":1009,"context_line":"                account, container, err.http_status)"},{"line_number":1010,"context_line":"        except (Exception, Timeout) as err:"},{"line_number":1011,"context_line":"            self.logger.exception("},{"line_number":1012,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"8326c97c_d2db830b","line":1009,"updated":"2022-07-13 01:14:45.000000000","message":"I like this, kinda feels like a drive by improvement, but also will help point us to the account container and to know it was deleted.\n\nI wonder if we should log anywhere that we\u0027re processing a deleted container, is that useful information? Probably doesn\u0027t matter.","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"994bef044be498e0527f87e6e0f778a710bcf943","unresolved":false,"context_lines":[{"line_number":1006,"context_line":"            self.logger.warning("},{"line_number":1007,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"},{"line_number":1008,"context_line":"                node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027],"},{"line_number":1009,"context_line":"                account, container, err.http_status)"},{"line_number":1010,"context_line":"        except (Exception, Timeout) as err:"},{"line_number":1011,"context_line":"            self.logger.exception("},{"line_number":1012,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"7c42490c_ca5075a4","line":1009,"in_reply_to":"380ab99f_9694b964","updated":"2022-07-27 14:50:44.000000000","message":"Done","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2c7178b34e48de0a167b6b4c824ca3c54e47e28a","unresolved":true,"context_lines":[{"line_number":1006,"context_line":"            self.logger.warning("},{"line_number":1007,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"},{"line_number":1008,"context_line":"                node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027],"},{"line_number":1009,"context_line":"                account, container, err.http_status)"},{"line_number":1010,"context_line":"        except (Exception, Timeout) as err:"},{"line_number":1011,"context_line":"            self.logger.exception("},{"line_number":1012,"context_line":"                \u0027Failed to put shard ranges to %s:%s/%s %s/%s: %s\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"380ab99f_9694b964","line":1009,"in_reply_to":"8326c97c_d2db830b","updated":"2022-07-23 05:18:54.000000000","message":"I wonder if we should quote them, though -- I\u0027m a little worried about reserved-namespace names.","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2c7178b34e48de0a167b6b4c824ca3c54e47e28a","unresolved":true,"context_lines":[{"line_number":2042,"context_line":""},{"line_number":2043,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self.auto_shard"},{"line_number":2044,"context_line":"        if state in (UNSHARDED, COLLAPSED):"},{"line_number":2045,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":2046,"context_line":"                # bootstrap sharding of root container"},{"line_number":2047,"context_line":"                self._find_and_enable_sharding_candidates("},{"line_number":2048,"context_line":"                    broker, shard_ranges\u003d[broker.get_own_shard_range()])"}],"source_content_type":"text/x-python","patch_set":9,"id":"c84578bf_a738d91b","line":2045,"updated":"2022-07-23 05:18:54.000000000","message":"Is it weird at all to have a deleted DB performing leader actions? It *feels* a little weird...","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"994bef044be498e0527f87e6e0f778a710bcf943","unresolved":false,"context_lines":[{"line_number":2042,"context_line":""},{"line_number":2043,"context_line":"        is_leader \u003d node[\u0027index\u0027] \u003d\u003d 0 and self.auto_shard"},{"line_number":2044,"context_line":"        if state in (UNSHARDED, COLLAPSED):"},{"line_number":2045,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":2046,"context_line":"                # bootstrap sharding of root container"},{"line_number":2047,"context_line":"                self._find_and_enable_sharding_candidates("},{"line_number":2048,"context_line":"                    broker, shard_ranges\u003d[broker.get_own_shard_range()])"}],"source_content_type":"text/x-python","patch_set":9,"id":"ea04ad81_79fbdc70","line":2045,"in_reply_to":"c84578bf_a738d91b","updated":"2022-07-27 14:50:44.000000000","message":"If the db is deleted, and therefore has no object rows, I don\u0027t think the leader actions would have any outcome, but happy to add a guard condition anyway","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2c7178b34e48de0a167b6b4c824ca3c54e47e28a","unresolved":true,"context_lines":[{"line_number":2068,"context_line":""},{"line_number":2069,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":2070,"context_line":"            if is_leader:"},{"line_number":2071,"context_line":"                num_found \u003d self._find_shard_ranges(broker)"},{"line_number":2072,"context_line":"            else:"},{"line_number":2073,"context_line":"                num_found \u003d 0"},{"line_number":2074,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"d7b2d374_b24f6c37","line":2071,"updated":"2022-07-23 05:18:54.000000000","message":"Again, having a deleted DB performing leader actions... seems suspect.","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"994bef044be498e0527f87e6e0f778a710bcf943","unresolved":false,"context_lines":[{"line_number":2068,"context_line":""},{"line_number":2069,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":2070,"context_line":"            if is_leader:"},{"line_number":2071,"context_line":"                num_found \u003d self._find_shard_ranges(broker)"},{"line_number":2072,"context_line":"            else:"},{"line_number":2073,"context_line":"                num_found \u003d 0"},{"line_number":2074,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"8ff42628_e94545bf","line":2071,"in_reply_to":"d7b2d374_b24f6c37","updated":"2022-07-27 14:50:44.000000000","message":"Done","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2c7178b34e48de0a167b6b4c824ca3c54e47e28a","unresolved":true,"context_lines":[{"line_number":2115,"context_line":"                self._update_root_container(broker)"},{"line_number":2116,"context_line":""},{"line_number":2117,"context_line":"        self.logger.debug(\u0027Finished processing %s state %s\u0027,"},{"line_number":2118,"context_line":"                          quote(broker.path), broker.get_db_state())"},{"line_number":2119,"context_line":""},{"line_number":2120,"context_line":"    def _one_shard_cycle(self, devices_to_shard, partitions_to_shard):"},{"line_number":2121,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"124eb005_27240545","line":2118,"updated":"2022-07-23 05:18:54.000000000","message":"I wonder if this (and the \"Starting...\" message at L2034) should call out that the broker is deleted... or maybe we just throw an\n\n else:\n     self.logger.debug(\u0027DB is deleted; not sending stats\u0027)\n\njust above?","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"994bef044be498e0527f87e6e0f778a710bcf943","unresolved":false,"context_lines":[{"line_number":2115,"context_line":"                self._update_root_container(broker)"},{"line_number":2116,"context_line":""},{"line_number":2117,"context_line":"        self.logger.debug(\u0027Finished processing %s state %s\u0027,"},{"line_number":2118,"context_line":"                          quote(broker.path), broker.get_db_state())"},{"line_number":2119,"context_line":""},{"line_number":2120,"context_line":"    def _one_shard_cycle(self, devices_to_shard, partitions_to_shard):"},{"line_number":2121,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"17cf6c10_566d6862","line":2118,"in_reply_to":"124eb005_27240545","updated":"2022-07-27 14:50:44.000000000","message":"Done","commit_id":"a303319f0c21cff5eef74a229d5131d3a6b0ef34"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d5066e6a20ba50220248f14108b4a5ceeac95853","unresolved":true,"context_lines":[{"line_number":3531,"context_line":"        self.brain.stop_handoff_half()"},{"line_number":3532,"context_line":"        obj_names \u003d self._make_object_names(8)"},{"line_number":3533,"context_line":"        self.put_objects(obj_names)"},{"line_number":3534,"context_line":"        self.brain.start_handoff_half()"},{"line_number":3535,"context_line":""},{"line_number":3536,"context_line":"        # run replicators first time to get sync points set"},{"line_number":3537,"context_line":"        self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":1,"id":"c67637ff_a24620ff","line":3534,"updated":"2022-07-13 01:14:45.000000000","message":"doesn\u0027t starting this again before the replicators run means the unsharded one without objects, gets the objects from the others?\n\nIf were just sharding 2 and then deleting objects we could probably just do that.. probably don\u0027t need to turn the unsharded off.\n\nIf you want to mark the unsharded as deleted because it has no objects while others do, then you want to replicate before starting the handoff half I believe.","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"994bef044be498e0527f87e6e0f778a710bcf943","unresolved":false,"context_lines":[{"line_number":3531,"context_line":"        self.brain.stop_handoff_half()"},{"line_number":3532,"context_line":"        obj_names \u003d self._make_object_names(8)"},{"line_number":3533,"context_line":"        self.put_objects(obj_names)"},{"line_number":3534,"context_line":"        self.brain.start_handoff_half()"},{"line_number":3535,"context_line":""},{"line_number":3536,"context_line":"        # run replicators first time to get sync points set"},{"line_number":3537,"context_line":"        self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":1,"id":"eb41ad29_dc794831","line":3534,"in_reply_to":"8523b64f_cde4efa4","updated":"2022-07-27 14:50:44.000000000","message":"Done","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf221bd56af3d10c77d211cdda3d227c5b96a064","unresolved":true,"context_lines":[{"line_number":3531,"context_line":"        self.brain.stop_handoff_half()"},{"line_number":3532,"context_line":"        obj_names \u003d self._make_object_names(8)"},{"line_number":3533,"context_line":"        self.put_objects(obj_names)"},{"line_number":3534,"context_line":"        self.brain.start_handoff_half()"},{"line_number":3535,"context_line":""},{"line_number":3536,"context_line":"        # run replicators first time to get sync points set"},{"line_number":3537,"context_line":"        self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":1,"id":"8523b64f_cde4efa4","line":3534,"in_reply_to":"c67637ff_a24620ff","updated":"2022-07-13 10:20:51.000000000","message":"good point! The objects get deleted later - I think I was concerned that the delete updates would got to shards and not roots and so the unsharded root needed to not get the original PUTs in order to be empty. But you\u0027re right, the replication before sharding would sync the object rows anyway, and the proxies will be caching out of date container info so delete updates do go to the root.","commit_id":"f365974e2255c40a6fb1e90750d0b8a7e984a0fe"}]}