)]}'
{"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":612,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":613,"context_line":"                # TODO: bring back leader election (maybe?)"},{"line_number":614,"context_line":"                if node[\u0027index\u0027] \u003d\u003d 0 and not scan_complete:"},{"line_number":615,"context_line":"                    if broker.get_db_state \u003d\u003d DB_STATE_UNSHARDED:"},{"line_number":616,"context_line":"                        # TODO (acoles): I *think* this is the right place to"},{"line_number":617,"context_line":"                        # transition to SHARDING state - we are about to scan"},{"line_number":618,"context_line":"                        # for pivots so want the original db closed to new"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_78cd295c","side":"PARENT","line":615,"updated":"2017-10-10 23:46:33.000000000","message":"Eeep. Good catch.","commit_id":"1434401590a4a72de11be2ae7f9dc2deeba07732"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":107,"context_line":"            raise ValueError(err.message + \": shard_shrink_merge_point\")"},{"line_number":108,"context_line":"        self.shard_container_size \u003d config_positive_int_value("},{"line_number":109,"context_line":"            conf.get(\u0027shard_container_size\u0027, 10000000))"},{"line_number":110,"context_line":"        # TODO: size below which shards may be shrunk"},{"line_number":111,"context_line":"        self.shrink_size \u003d self.shard_container_size * self.shard_shrink_point"},{"line_number":112,"context_line":"        # TODO: size below which shards will accept a merge"},{"line_number":113,"context_line":"        self.merge_size \u003d self.shard_container_size * self.shrink_merge_point"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_fc59ed59","line":110,"range":{"start_line":110,"start_character":8,"end_line":110,"end_character":15},"updated":"2017-10-11 18:32:18.000000000","message":"heh. so many of my comments start with TODO that I think this was reflex - these are not TODO\u0027s","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5263c231381f2da256c0bb13c09add7566a60378","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                # This container is deleted so we can skip it. We still want"},{"line_number":572,"context_line":"                # deleted containers to go via misplaced items, cause they may"},{"line_number":573,"context_line":"                # have new objects sitting in them that may need to move."},{"line_number":574,"context_line":"                continue"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"            state \u003d broker.get_db_state()"},{"line_number":577,"context_line":"            if state \u003d\u003d DB_STATE_NOTFOUND:"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_0128f983","line":574,"updated":"2017-10-11 04:33:22.000000000","message":"So if I remember correctly `is_deleted` wont return true if the delete ts \u003e put ts and there are still objects in the object table.\n\nMisplaced objects _should_ move them all. But potentially there is a chance that we could have a recently cleaved shard that is marked deleted, but get passed here with objects and then tries to shrink any new shards it\u0027s created, well scan for them, it shouldn\u0027t shrink.\n\nAnyway, I think there is a chance we could scan non root containers here. Which although _should_ be safe. But I don\u0027t think want to scan unless we have to. So maybe we should use an is_root_container check?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":581,"context_line":""},{"line_number":582,"context_line":"            try:"},{"line_number":583,"context_line":"                if state \u003d\u003d DB_STATE_SHARDED and node[\u0027index\u0027] \u003d\u003d 0:"},{"line_number":584,"context_line":"                        self._find_shrinks(broker, node, part)"},{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_18dd1514","line":584,"updated":"2017-10-10 23:46:33.000000000","message":"Dedent. Should there be a continue after this? We\u0027ve already dealt with misplaced objects... scanning\u0027s already done... shouldn\u0027t have a giant DB from which to cleave...","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":581,"context_line":""},{"line_number":582,"context_line":"            try:"},{"line_number":583,"context_line":"                if state \u003d\u003d DB_STATE_SHARDED and node[\u0027index\u0027] \u003d\u003d 0:"},{"line_number":584,"context_line":"                        self._find_shrinks(broker, node, part)"},{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_f059165a","line":584,"in_reply_to":"5f4e5783_18dd1514","updated":"2017-10-11 18:32:18.000000000","message":"yes, I think you are right. I remember being concerned about the finally clause being executed...but looking again it should not be needed\n\nActually, the state conditions needed cleaning up - we can continue here on any node, but only for a root container - shard container need to update usage stats (in finally clause)","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":583,"context_line":"                if state \u003d\u003d DB_STATE_SHARDED and node[\u0027index\u0027] \u003d\u003d 0:"},{"line_number":584,"context_line":"                        self._find_shrinks(broker, node, part)"},{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."},{"line_number":588,"context_line":"                object_count \u003d broker.get_info()[\u0027object_count\u0027]"},{"line_number":589,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":590,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_787ac924","line":587,"range":{"start_line":586,"start_character":18,"end_line":587,"end_character":27},"updated":"2017-10-10 23:46:33.000000000","message":"Needs updating, if we\u0027re not going to continue above.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."},{"line_number":588,"context_line":"                object_count \u003d broker.get_info()[\u0027object_count\u0027]"},{"line_number":589,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":590,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":591,"context_line":"                # TODO: bring back leader election (maybe?)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_b89de15a","line":588,"updated":"2017-10-10 23:46:33.000000000","message":"What\u0027s this look like when we\u0027re in the SHARDED state? Doesn\u0027t it come from a\n\n SELECT SUM(object_count) FROM shard_ranges WHERE deleted\u003d0\n\nor somesuch now?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."},{"line_number":588,"context_line":"                object_count \u003d broker.get_info()[\u0027object_count\u0027]"},{"line_number":589,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":590,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":591,"context_line":"                # TODO: bring back leader election (maybe?)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_10095230","line":588,"in_reply_to":"5f4e5783_816989bf","updated":"2017-10-11 18:32:18.000000000","message":"as above - we need a clean up of the condition through this method e.g. to not ever get here when already sharded","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5263c231381f2da256c0bb13c09add7566a60378","unresolved":false,"context_lines":[{"line_number":585,"context_line":""},{"line_number":586,"context_line":"                # We are either in the sharding state or we need to start"},{"line_number":587,"context_line":"                # sharding."},{"line_number":588,"context_line":"                object_count \u003d broker.get_info()[\u0027object_count\u0027]"},{"line_number":589,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":590,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":591,"context_line":"                # TODO: bring back leader election (maybe?)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_816989bf","line":588,"in_reply_to":"5f4e5783_b89de15a","updated":"2017-10-11 04:33:22.000000000","message":"Yup, so this may start trying to shard a root container, though fail at finding shard_ranges.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":799,"context_line":"        # a neighbour. We may need to expose row count as well as object count."},{"line_number":800,"context_line":"        state \u003d broker.get_db_state()"},{"line_number":801,"context_line":"        if state !\u003d DB_STATE_SHARDED:"},{"line_number":802,"context_line":"            self.logger.debug(\u0027Shrink: ignoring not sharded container %s/%s\u0027,"},{"line_number":803,"context_line":"                              broker.account, broker.container)"},{"line_number":804,"context_line":"            return"},{"line_number":805,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_1814f59f","line":802,"range":{"start_line":802,"start_character":24,"end_line":802,"end_character":29},"updated":"2017-10-10 23:46:33.000000000","message":"Warning might be better -- isn\u0027t there just the one caller? And it already has a db_state check.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":799,"context_line":"        # a neighbour. We may need to expose row count as well as object count."},{"line_number":800,"context_line":"        state \u003d broker.get_db_state()"},{"line_number":801,"context_line":"        if state !\u003d DB_STATE_SHARDED:"},{"line_number":802,"context_line":"            self.logger.debug(\u0027Shrink: ignoring not sharded container %s/%s\u0027,"},{"line_number":803,"context_line":"                              broker.account, broker.container)"},{"line_number":804,"context_line":"            return"},{"line_number":805,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_130efc31","line":802,"range":{"start_line":802,"start_character":24,"end_line":802,"end_character":29},"in_reply_to":"5f4e5783_1814f59f","updated":"2017-10-11 18:32:18.000000000","message":"Done","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":810,"context_line":"        for sr in shard_ranges:"},{"line_number":811,"context_line":"            self.logger.debug(\u0027shrink start %s\u0027 % dict(sr))"},{"line_number":812,"context_line":"        expanding_ranges \u003d {}"},{"line_number":813,"context_line":"        for i in range(len(shard_ranges) - 1):"},{"line_number":814,"context_line":"            donor \u003d shard_ranges[i]"},{"line_number":815,"context_line":"            if donor in expanding_ranges:"},{"line_number":816,"context_line":"                # we may have previously expanded this range; if so then"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_f8c239e9","line":813,"updated":"2017-10-10 23:46:33.000000000","message":"nit:\n\n for donor, acceptor in zip(shard_ranges, shard_range[1:]):\n\n?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":810,"context_line":"        for sr in shard_ranges:"},{"line_number":811,"context_line":"            self.logger.debug(\u0027shrink start %s\u0027 % dict(sr))"},{"line_number":812,"context_line":"        expanding_ranges \u003d {}"},{"line_number":813,"context_line":"        for i in range(len(shard_ranges) - 1):"},{"line_number":814,"context_line":"            donor \u003d shard_ranges[i]"},{"line_number":815,"context_line":"            if donor in expanding_ranges:"},{"line_number":816,"context_line":"                # we may have previously expanded this range; if so then"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_f36360db","line":813,"in_reply_to":"5f4e5783_f8c239e9","updated":"2017-10-11 18:32:18.000000000","message":"nice!","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":824,"context_line":""},{"line_number":825,"context_line":"            acceptor \u003d shard_ranges[i + 1]"},{"line_number":826,"context_line":"            if (donor.object_count \u003c self.shrink_size and"},{"line_number":827,"context_line":"                    acceptor.object_count \u003c self.merge_size):"},{"line_number":828,"context_line":"                expanding_ranges[acceptor] \u003d donor"},{"line_number":829,"context_line":"                self.logger.debug(\u0027shrink %s can be merged to %s\u0027 %"},{"line_number":830,"context_line":"                                  (donor, acceptor))"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_b8ee0186","line":827,"range":{"start_line":827,"start_character":20,"end_line":827,"end_character":41},"updated":"2017-10-10 23:46:33.000000000","message":"Maybe better as\n\n donor.object_count + acceptor.object_count\n\n? I kinda want to be hesitant to shrink, but maybe all I\u0027m really doing is\n\n merge_size :\u003d merge_size - shrink_size\n\nBut since they\u0027re both bounded to [0, shard_container_size]... seems like a good/reasonable thing?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5263c231381f2da256c0bb13c09add7566a60378","unresolved":false,"context_lines":[{"line_number":824,"context_line":""},{"line_number":825,"context_line":"            acceptor \u003d shard_ranges[i + 1]"},{"line_number":826,"context_line":"            if (donor.object_count \u003c self.shrink_size and"},{"line_number":827,"context_line":"                    acceptor.object_count \u003c self.merge_size):"},{"line_number":828,"context_line":"                expanding_ranges[acceptor] \u003d donor"},{"line_number":829,"context_line":"                self.logger.debug(\u0027shrink %s can be merged to %s\u0027 %"},{"line_number":830,"context_line":"                                  (donor, acceptor))"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_c159c1fe","line":827,"range":{"start_line":827,"start_character":20,"end_line":827,"end_character":41},"in_reply_to":"5f4e5783_b8ee0186","updated":"2017-10-11 04:33:22.000000000","message":"In the old version it was:\n\n  accepter.object_count + donor.object_count \u003c self.merge_size","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":824,"context_line":""},{"line_number":825,"context_line":"            acceptor \u003d shard_ranges[i + 1]"},{"line_number":826,"context_line":"            if (donor.object_count \u003c self.shrink_size and"},{"line_number":827,"context_line":"                    acceptor.object_count \u003c self.merge_size):"},{"line_number":828,"context_line":"                expanding_ranges[acceptor] \u003d donor"},{"line_number":829,"context_line":"                self.logger.debug(\u0027shrink %s can be merged to %s\u0027 %"},{"line_number":830,"context_line":"                                  (donor, acceptor))"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_7c285d33","line":827,"range":{"start_line":827,"start_character":20,"end_line":827,"end_character":41},"in_reply_to":"5f4e5783_c159c1fe","updated":"2017-10-11 18:32:18.000000000","message":"Done","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":852,"context_line":"            # ignored."},{"line_number":853,"context_line":"            # 3. PUT new shard lower bound to the acceptor shard container so"},{"line_number":854,"context_line":"            # that it will start to accept any updates for its new expanded"},{"line_number":855,"context_line":"            # range, but do not update its timestamp so that 2 holds true."},{"line_number":856,"context_line":"            # 4. Send expanded acceptor shard range to retiring donor, with new"},{"line_number":857,"context_line":"            # timestamp."},{"line_number":858,"context_line":"            # 5. Asynchronously, donor shard cleaves and replicates object"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_23e9c219","line":855,"updated":"2017-10-10 23:46:33.000000000","message":"We should probably have _misplaced_objects get the latest shard ranges earlier when we\u0027re a shard -- then if this fails, the shard can update itself *before* deciding on queries. Donor should look, see that it shouldn\u0027t exist any more; acceptor should look, see that its range needs to expand.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":854,"context_line":"            # that it will start to accept any updates for its new expanded"},{"line_number":855,"context_line":"            # range, but do not update its timestamp so that 2 holds true."},{"line_number":856,"context_line":"            # 4. Send expanded acceptor shard range to retiring donor, with new"},{"line_number":857,"context_line":"            # timestamp."},{"line_number":858,"context_line":"            # 5. Asynchronously, donor shard cleaves and replicates object"},{"line_number":859,"context_line":"            # rows and shard range metadata to acceptor container, including"},{"line_number":860,"context_line":"            # new shard range timestamp."}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_a3dc9274","line":857,"updated":"2017-10-10 23:46:33.000000000","message":"Eh? Why? Do we generally expect shards to have a good idea of what the other shards are?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":854,"context_line":"            # that it will start to accept any updates for its new expanded"},{"line_number":855,"context_line":"            # range, but do not update its timestamp so that 2 holds true."},{"line_number":856,"context_line":"            # 4. Send expanded acceptor shard range to retiring donor, with new"},{"line_number":857,"context_line":"            # timestamp."},{"line_number":858,"context_line":"            # 5. Asynchronously, donor shard cleaves and replicates object"},{"line_number":859,"context_line":"            # rows and shard range metadata to acceptor container, including"},{"line_number":860,"context_line":"            # new shard range timestamp."}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_3cd0a51f","line":857,"in_reply_to":"5f4e5783_4177113c","updated":"2017-10-11 18:32:18.000000000","message":"What Matt said - this is what triggers the donor to cleave all its contents to the acceptor shard - because the acceptor shard is put into the donor\u0027s shard_ranges.\n\nAs Matt notes, I still need to figure out deleting. Also, there is a corner case where the root does this while the donor is in fact shrading itself (if the root and donor object counts are way out of sync).","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5263c231381f2da256c0bb13c09add7566a60378","unresolved":false,"context_lines":[{"line_number":854,"context_line":"            # that it will start to accept any updates for its new expanded"},{"line_number":855,"context_line":"            # range, but do not update its timestamp so that 2 holds true."},{"line_number":856,"context_line":"            # 4. Send expanded acceptor shard range to retiring donor, with new"},{"line_number":857,"context_line":"            # timestamp."},{"line_number":858,"context_line":"            # 5. Asynchronously, donor shard cleaves and replicates object"},{"line_number":859,"context_line":"            # rows and shard range metadata to acceptor container, including"},{"line_number":860,"context_line":"            # new shard range timestamp."}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_4177113c","line":857,"in_reply_to":"5f4e5783_a3dc9274","updated":"2017-10-11 04:33:22.000000000","message":"The idea I think here, is by sending the acceptor range to the doner. The cleaving method will find a range it\u0027ll need to \"split\" on.. and all its records will be selected and goto the acceptor. Pretty cleaver.\n\nIt needs the exact name to use (so the acceptors shard_range) so it can make a handoff db that will be replicated into the acceptor.\n\nIf you also somehow send a scan complete header to it, will the doner also delete itself when done? (ie jump to sharding -\u003e sharded -\u003e deleted).","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":872,"context_line":"            # range - between now and that happening, where do updates for the"},{"line_number":873,"context_line":"            # retired range get directed?"},{"line_number":874,"context_line":"            acceptor.lower \u003d donor.lower"},{"line_number":875,"context_line":"            self._put_shard_container(broker, acceptor)"},{"line_number":876,"context_line":"            # give the acceptor a fresh timestamp so that the only shard range"},{"line_number":877,"context_line":"            # updates accepted at the root will be for the expanded acceptor"},{"line_number":878,"context_line":"            # shard range once it receives its new timestamp via replication"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_631b3a1b","line":875,"updated":"2017-10-10 23:46:33.000000000","message":"When this fails, does _put_shard_container swallow the error, or do we bomb out pretty hard?","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":872,"context_line":"            # range - between now and that happening, where do updates for the"},{"line_number":873,"context_line":"            # retired range get directed?"},{"line_number":874,"context_line":"            acceptor.lower \u003d donor.lower"},{"line_number":875,"context_line":"            self._put_shard_container(broker, acceptor)"},{"line_number":876,"context_line":"            # give the acceptor a fresh timestamp so that the only shard range"},{"line_number":877,"context_line":"            # updates accepted at the root will be for the expanded acceptor"},{"line_number":878,"context_line":"            # shard range once it receives its new timestamp via replication"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_5c5fe171","line":875,"in_reply_to":"5f4e5783_631b3a1b","updated":"2017-10-11 18:32:18.000000000","message":"I think the process survives us swallowing this error - the donor will update the acceptor\u0027s upper/lower when it replicates to it during cleave. The choice is probably driven by what happens to updates during the transition - we don\u0027t want the acceptor treating updates for the new range as being misplaced because it didn\u0027t find out about the new range.\n\nIf we fail to update the acceptor range before the shrink occurs... misplaced_objects does consult root for the correct location of objects - so we should check that the correct location is not in fact the current broker before misplaced_objects replicates to itself!\n\nOr we just bomb out here if the update fails - the cost of getting to this point is small, if the acceptor container put failed maybe it is not the best time to be merging objects to it???","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":904,"context_line":"            self._put_shard_container(broker, donor)"},{"line_number":905,"context_line":"            # TODO: not sure we really want to delete the donor yet but need"},{"line_number":906,"context_line":"            # some way to ensure it is passed over in future shrink cycles;"},{"line_number":907,"context_line":"            # some audit process should delete the shard once it has emptied"},{"line_number":908,"context_line":"            donor.deleted \u003d 1"},{"line_number":909,"context_line":"            updated_shard_ranges.append(donor)"},{"line_number":910,"context_line":"            updated_shard_ranges.append(acceptor)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_63f0daca","line":907,"updated":"2017-10-10 23:46:33.000000000","message":"I think this could also be a signal for the donor to cleave.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":905,"context_line":"            # TODO: not sure we really want to delete the donor yet but need"},{"line_number":906,"context_line":"            # some way to ensure it is passed over in future shrink cycles;"},{"line_number":907,"context_line":"            # some audit process should delete the shard once it has emptied"},{"line_number":908,"context_line":"            donor.deleted \u003d 1"},{"line_number":909,"context_line":"            updated_shard_ranges.append(donor)"},{"line_number":910,"context_line":"            updated_shard_ranges.append(acceptor)"},{"line_number":911,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_63ab3a1d","line":908,"updated":"2017-10-10 23:46:33.000000000","message":"I wonder if we should be updating\n\n acceptor.object_count +\u003d donor.object_count\n\nso we don\u0027t try to shrink N small-but-non-zero donors into a just-barely-under-the-threshold acceptor while the donors\u0027 sharders aren\u0027t running.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":905,"context_line":"            # TODO: not sure we really want to delete the donor yet but need"},{"line_number":906,"context_line":"            # some way to ensure it is passed over in future shrink cycles;"},{"line_number":907,"context_line":"            # some audit process should delete the shard once it has emptied"},{"line_number":908,"context_line":"            donor.deleted \u003d 1"},{"line_number":909,"context_line":"            updated_shard_ranges.append(donor)"},{"line_number":910,"context_line":"            updated_shard_ranges.append(acceptor)"},{"line_number":911,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_3c89c5c9","line":908,"in_reply_to":"5f4e5783_63ab3a1d","updated":"2017-10-11 18:32:18.000000000","message":"yes! see line 888 - I had it commented to try to convince myself that the acceptor shard container was updating root with its new usage and not just being short circuited.\n\nyour point about another donor merging to this acceptor on subsequent cycle is interesting to ponder too...I backed out a load of code I had to merge multiple donors to same acceptor in one cycle because (a) KISS and (b) I hope that single donor merging to single acceptor gives us some means to know when the acceptor has been expanded. If the next cycle generates another donor to same acceptor before first is done...could be tricky - needs more thought!","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":917,"context_line":"            # if any mods were made to broker"},{"line_number":918,"context_line":"            self.cpool.spawn("},{"line_number":919,"context_line":"                self._replicate_object, part, broker.db_file, node[\u0027id\u0027])"},{"line_number":920,"context_line":"            # any(self.cpool)"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"        self.logger.debug(\u0027shrink done, modified %s\u0027 % updated_shard_ranges)"},{"line_number":923,"context_line":"        for sr in broker.get_shard_ranges():"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_a35ff23f","line":920,"updated":"2017-10-10 23:46:33.000000000","message":"??","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":917,"context_line":"            # if any mods were made to broker"},{"line_number":918,"context_line":"            self.cpool.spawn("},{"line_number":919,"context_line":"                self._replicate_object, part, broker.db_file, node[\u0027id\u0027])"},{"line_number":920,"context_line":"            # any(self.cpool)"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"        self.logger.debug(\u0027shrink done, modified %s\u0027 % updated_shard_ranges)"},{"line_number":923,"context_line":"        for sr in broker.get_shard_ranges():"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_dc7bd1bb","line":920,"in_reply_to":"5f4e5783_a35ff23f","updated":"2017-10-11 18:32:18.000000000","message":":/ we should wait right? but we don\u0027t in find_shard_ranges #confused","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e143f1a6f2a19c92a4118280ed4343f7a9bbfd67","unresolved":false,"context_lines":[{"line_number":550,"context_line":"                    # it\u0027s acceptor we set it\u0027s object count to the actual"},{"line_number":551,"context_line":"                    # content of the new cleave broker, which is only the"},{"line_number":552,"context_line":"                    # objects being merged form the donor, so is gauranteed to"},{"line_number":553,"context_line":"                    # be less than the new total for the acceptor shard!"},{"line_number":554,"context_line":"                    # TODO: these shard ranges may have existed for some time"},{"line_number":555,"context_line":"                    # and already be updating the root with their usage e.g."},{"line_number":556,"context_line":"                    # multiple cycles before we finished cleaving, or process"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_fd57cc25","line":553,"updated":"2017-12-13 19:44:07.000000000","message":"Hmm? If we don\u0027t do it here, when does the root find out about the newly-defined shards? Do we just rely on them reporting back to the root later, when their own sharder daemon runs?\n\nIf we don\u0027t tell the root about the new shards but we *do* mark our own shard as deleted, how does the root deal with the gap in the namespace?","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"46632e667fed275eb2d6d0637d911038323b5513","unresolved":false,"context_lines":[{"line_number":550,"context_line":"                    # it\u0027s acceptor we set it\u0027s object count to the actual"},{"line_number":551,"context_line":"                    # content of the new cleave broker, which is only the"},{"line_number":552,"context_line":"                    # objects being merged form the donor, so is gauranteed to"},{"line_number":553,"context_line":"                    # be less than the new total for the acceptor shard!"},{"line_number":554,"context_line":"                    # TODO: these shard ranges may have existed for some time"},{"line_number":555,"context_line":"                    # and already be updating the root with their usage e.g."},{"line_number":556,"context_line":"                    # multiple cycles before we finished cleaving, or process"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_f0157be6","line":553,"in_reply_to":"df87a7cf_3ffe7505","updated":"2017-12-15 16:45:01.000000000","message":"In last comment I was focussed on shrinking, but when a shard is sharding itself these updates to the root container *are* significant. We should probably figure out a way to differentiate the two cases, or make sure choice of timestamps and meta timestamps results in the correct outcome at the root.","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"12dc516c0e42d02f1832bbff96f5546eb179fb68","unresolved":false,"context_lines":[{"line_number":550,"context_line":"                    # it\u0027s acceptor we set it\u0027s object count to the actual"},{"line_number":551,"context_line":"                    # content of the new cleave broker, which is only the"},{"line_number":552,"context_line":"                    # objects being merged form the donor, so is gauranteed to"},{"line_number":553,"context_line":"                    # be less than the new total for the acceptor shard!"},{"line_number":554,"context_line":"                    # TODO: these shard ranges may have existed for some time"},{"line_number":555,"context_line":"                    # and already be updating the root with their usage e.g."},{"line_number":556,"context_line":"                    # multiple cycles before we finished cleaving, or process"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_c9752efc","line":553,"in_reply_to":"df87a7cf_f0157be6","updated":"2017-12-15 18:49:05.000000000","message":"Yeah, I was thinking about the split case...\n\nMaybe we could look at broker.get_shard_ranges(), and skip the update iff there is a shard that is a strict super-set of our own range?","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"027a093443571f5e1f1feb13b73627be988ebbb2","unresolved":false,"context_lines":[{"line_number":550,"context_line":"                    # it\u0027s acceptor we set it\u0027s object count to the actual"},{"line_number":551,"context_line":"                    # content of the new cleave broker, which is only the"},{"line_number":552,"context_line":"                    # objects being merged form the donor, so is gauranteed to"},{"line_number":553,"context_line":"                    # be less than the new total for the acceptor shard!"},{"line_number":554,"context_line":"                    # TODO: these shard ranges may have existed for some time"},{"line_number":555,"context_line":"                    # and already be updating the root with their usage e.g."},{"line_number":556,"context_line":"                    # multiple cycles before we finished cleaving, or process"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_3ffe7505","line":553,"in_reply_to":"df87a7cf_fd57cc25","updated":"2017-12-14 12:35:17.000000000","message":"My reasoning was that the acceptor shard range already exists and already has an expanded namepsace in the root container db, so no need to update it here:\n\n- root expands acceptor namespace, and PUTs that to the acceptor shard container\n- root then pushes the acceptor shard range to the donor\n- donor cleaves to the acceptor by generating a local (misplaced) acceptor db to replicate\n- no need to tell the root about the acceptor shard range, root already knows about it\n- but donor version of acceptor db has donor\u0027s object count which is almost certainly less than the acceptor\u0027s real object count\n- so perhaps it is pointless to send root an update here?\n\nLet\u0027s leave the comments for now.","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"}],"test/probe/common.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e143f1a6f2a19c92a4118280ed4343f7a9bbfd67","unresolved":false,"context_lines":[{"line_number":485,"context_line":"        try:"},{"line_number":486,"context_line":"            yield"},{"line_number":487,"context_line":"        except AssertionError as err:"},{"line_number":488,"context_line":"            self.fail(\u0027%s Failed with %s\u0027 % (msg, err))"},{"line_number":489,"context_line":""},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"class ReplProbeTest(ProbeTest):"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_fd058c15","line":488,"updated":"2017-12-13 19:44:07.000000000","message":"I rather like this -- could be handy in some loop-based, data-driven tests. Might want it for unit tests, too :-)","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"027a093443571f5e1f1feb13b73627be988ebbb2","unresolved":false,"context_lines":[{"line_number":485,"context_line":"        try:"},{"line_number":486,"context_line":"            yield"},{"line_number":487,"context_line":"        except AssertionError as err:"},{"line_number":488,"context_line":"            self.fail(\u0027%s Failed with %s\u0027 % (msg, err))"},{"line_number":489,"context_line":""},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"class ReplProbeTest(ProbeTest):"}],"source_content_type":"text/x-python","patch_set":4,"id":"df87a7cf_512d7244","line":488,"in_reply_to":"df87a7cf_fd058c15","updated":"2017-12-14 12:35:17.000000000","message":"I have just added similar to test/unit/__init__.py in another patch that will soon be on feature/deep:\nhttps://review.openstack.org/#/c/527938/1/test/unit/__init__.py\n\nTBH I copied Clay\u0027s pattern here: https://github.com/openstack/swift/blob/e109c7800fcd22c48800cd6e18943b32b49d5e0b/test/probe/test_reconstructor_rebuild.py#L197-L207","commit_id":"60444688dbc7982bc8672f14eae221b9072fc310"}],"test/probe/test_sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"60e7cafc9190a91ddbb40c41bd0a96dc441e9331","unresolved":false,"context_lines":[{"line_number":61,"context_line":"        self.sharders \u003d Manager([\u0027container-sharder\u0027])"},{"line_number":62,"context_line":"        self.internal_client \u003d self.make_internal_client()"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone):"},{"line_number":65,"context_line":"        account \u003d account if account else self.account"},{"line_number":66,"context_line":"        container \u003d container if container else self.container_name"},{"line_number":67,"context_line":"        path \u003d self.internal_client.make_path(account, container)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_1eff3d02","line":64,"updated":"2017-10-10 23:46:33.000000000","message":"Oh good, I\u0027ve been thinking that something like this would be rather useful :-)\n\nOr even a direct_client, so we can make stronger assertions about when all replicas are in the same state.","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7c2de78036234a864f2c3a2142964d7b8792340f","unresolved":false,"context_lines":[{"line_number":61,"context_line":"        self.sharders \u003d Manager([\u0027container-sharder\u0027])"},{"line_number":62,"context_line":"        self.internal_client \u003d self.make_internal_client()"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone):"},{"line_number":65,"context_line":"        account \u003d account if account else self.account"},{"line_number":66,"context_line":"        container \u003d container if container else self.container_name"},{"line_number":67,"context_line":"        path \u003d self.internal_client.make_path(account, container)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_47138a5d","line":64,"in_reply_to":"5f4e5783_1eff3d02","updated":"2017-10-11 18:32:18.000000000","message":"I anticipate a direct client helper would be v useful\n\nI have also been dreaming of a cli tool that will show me relationships between all container dbs (e.g. during a stepped probe test), wth their metadata, \n\n`swift-shard-tree`\n\n ... and at end of its output show me orphan shards\n\nwould save tedious use of swift-container-info","commit_id":"ba1185a62820177c2a40d39486871068bbebcf4a"}]}