)]}'
{"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":505,"context_line":"        # parent. If its in both great, If it exists in either but not the"},{"line_number":506,"context_line":"        # other, then this needs to be fixed. If, however, it doesn\u0027t"},{"line_number":507,"context_line":"        # exist in either then this container may not exist anymore so"},{"line_number":508,"context_line":"        # quarantine it."},{"line_number":509,"context_line":"        # if not self._audit_shard_container(broker, shard_range,"},{"line_number":510,"context_line":"        #                                    root_account,"},{"line_number":511,"context_line":"        #                                    root_container):"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_3a3529f8","line":508,"updated":"2017-10-12 17:29:27.000000000","message":"Hmmm... right... this...","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":522,"context_line":"            return"},{"line_number":523,"context_line":""},{"line_number":524,"context_line":"        state \u003d broker.get_db_state()"},{"line_number":525,"context_line":"        if state \u003d\u003d DB_STATE_NOTFOUND:"},{"line_number":526,"context_line":"            return"},{"line_number":527,"context_line":""},{"line_number":528,"context_line":"        self.shard_cleanups \u003d dict()"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_ba091932","line":525,"range":{"start_line":525,"start_character":11,"end_line":525,"end_character":37},"updated":"2017-10-12 17:29:27.000000000","message":"Off topic: It seems a little strange to me that this isn\u0027t covered by broker.is_deleted() :-/","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":529,"context_line":"        # TODO: bring back leader election (maybe?); if so make it"},{"line_number":530,"context_line":"        # on-demand since we may not need to know if we are leader for all"},{"line_number":531,"context_line":"        # states"},{"line_number":532,"context_line":"        leader \u003d node[\u0027index\u0027] \u003d\u003d 0"},{"line_number":533,"context_line":"        try:"},{"line_number":534,"context_line":""},{"line_number":535,"context_line":"            if state \u003d\u003d DB_STATE_UNSHARDED:"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_c5aab6d0","line":532,"range":{"start_line":532,"start_character":8,"end_line":532,"end_character":14},"updated":"2017-10-12 17:29:27.000000000","message":"nit: is_leader?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":529,"context_line":"        # TODO: bring back leader election (maybe?); if so make it"},{"line_number":530,"context_line":"        # on-demand since we may not need to know if we are leader for all"},{"line_number":531,"context_line":"        # states"},{"line_number":532,"context_line":"        leader \u003d node[\u0027index\u0027] \u003d\u003d 0"},{"line_number":533,"context_line":"        try:"},{"line_number":534,"context_line":""},{"line_number":535,"context_line":"            if state \u003d\u003d DB_STATE_UNSHARDED:"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_01078b84","line":532,"range":{"start_line":532,"start_character":8,"end_line":532,"end_character":14},"in_reply_to":"5f4e5783_c5aab6d0","updated":"2017-10-13 10:30:00.000000000","message":"+1","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":549,"context_line":""},{"line_number":550,"context_line":"            if state \u003d\u003d DB_STATE_SHARDING:"},{"line_number":551,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":552,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":553,"context_line":"                if leader and not scan_complete:"},{"line_number":554,"context_line":"                    scan_complete \u003d self._find_shard_ranges("},{"line_number":555,"context_line":"                        broker, node, part)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_25e86ab2","line":552,"updated":"2017-10-12 17:29:27.000000000","message":"If we push pivots to donors when shrinking, we should probably push Scan-Done as well...","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":549,"context_line":""},{"line_number":550,"context_line":"            if state \u003d\u003d DB_STATE_SHARDING:"},{"line_number":551,"context_line":"                scan_complete \u003d config_true_value("},{"line_number":552,"context_line":"                    get_sharding_info(broker, \u0027Scan-Done\u0027))"},{"line_number":553,"context_line":"                if leader and not scan_complete:"},{"line_number":554,"context_line":"                    scan_complete \u003d self._find_shard_ranges("},{"line_number":555,"context_line":"                        broker, node, part)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_81d13bfe","line":552,"in_reply_to":"5f4e5783_25e86ab2","updated":"2017-10-13 10:30:00.000000000","message":"+1","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":567,"context_line":"                    self.logger.increment(\u0027sharding_complete\u0027)"},{"line_number":568,"context_line":""},{"line_number":569,"context_line":"                self._update_shard_range_counts("},{"line_number":570,"context_line":"                    broker, root_account, root_container)"},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"            if state \u003d\u003d DB_STATE_SHARDED:"},{"line_number":573,"context_line":"                if not broker.is_root_container():"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_25298a37","line":570,"updated":"2017-10-12 17:29:27.000000000","message":"So if we transitioned from unsharded to sharding above, we do this twice in one cycle?\n\nAlso -- if we\u0027re a shard that\u0027s splitting, what\u0027s this update look like? What if we just *finished* splitting?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":567,"context_line":"                    self.logger.increment(\u0027sharding_complete\u0027)"},{"line_number":568,"context_line":""},{"line_number":569,"context_line":"                self._update_shard_range_counts("},{"line_number":570,"context_line":"                    broker, root_account, root_container)"},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"            if state \u003d\u003d DB_STATE_SHARDED:"},{"line_number":573,"context_line":"                if not broker.is_root_container():"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_e12eaff8","line":570,"in_reply_to":"5f4e5783_25298a37","updated":"2017-10-13 10:30:00.000000000","message":"good catch - needs more conditions for the updates\n\nhmmm, if this is a sharding shard then it\u0027s about to be deleted so maybe no point updating the usage stats?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":570,"context_line":"                    broker, root_account, root_container)"},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"            if state \u003d\u003d DB_STATE_SHARDED:"},{"line_number":573,"context_line":"                if not broker.is_root_container():"},{"line_number":574,"context_line":"                    # sharded shard containers get cleaned up"},{"line_number":575,"context_line":"                    self.logger.info(\u0027Deleting sharded shard %s/%s\u0027,"},{"line_number":576,"context_line":"                                     broker.account, broker.container)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_20acd89e","line":573,"updated":"2017-10-12 17:29:27.000000000","message":"else, we\u0027ll insert some shrinking logic, yeah?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":570,"context_line":"                    broker, root_account, root_container)"},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"            if state \u003d\u003d DB_STATE_SHARDED:"},{"line_number":573,"context_line":"                if not broker.is_root_container():"},{"line_number":574,"context_line":"                    # sharded shard containers get cleaned up"},{"line_number":575,"context_line":"                    self.logger.info(\u0027Deleting sharded shard %s/%s\u0027,"},{"line_number":576,"context_line":"                                     broker.account, broker.container)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_a111d7ae","line":573,"in_reply_to":"5f4e5783_20acd89e","updated":"2017-10-13 10:30:00.000000000","message":"exactly. in fact this patch is backwards engineered from my shrink patch :)","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":574,"context_line":"                    # sharded shard containers get cleaned up"},{"line_number":575,"context_line":"                    self.logger.info(\u0027Deleting sharded shard %s/%s\u0027,"},{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_c0e2a401","line":577,"updated":"2017-10-12 17:29:27.000000000","message":"...are we in our *own* set of shard ranges?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6408541ab94c9319b20ca19497561df82c7993b2","unresolved":false,"context_lines":[{"line_number":574,"context_line":"                    # sharded shard containers get cleaned up"},{"line_number":575,"context_line":"                    self.logger.info(\u0027Deleting sharded shard %s/%s\u0027,"},{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_042af07d","line":577,"in_reply_to":"5f4e5783_816dbb33","updated":"2017-10-13 15:22:19.000000000","message":"I just got to thinking about how we\u0027ve been talking about having shard ranges inside of shard DBs... which we\u0027ll *have* to have when splitting a shard... and it seems like it could be a useful way to tell a donor shard that it needs to ship itself to an acceptor when shrinking...\n\nThis particular comment had me wonder: does a shard show up in its own set of shard ranges? Does it go and mark itself deleted when it\u0027s splitting and finds all its new ranges? How much does it need to track such things?\n\nNow I\u0027m thinking -- should we be replicating shard ranges across all DBs involved in this container? It\u0027d be a bit much to have root containers pushing out to all shards -- but maybe we could have the shards pull from the roots -- probably in the currently-commented-out _audit_shard_container? (We should probably rename that -- \"audit\" currently has a connotation of \"can be run offline\"...)","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":574,"context_line":"                    # sharded shard containers get cleaned up"},{"line_number":575,"context_line":"                    self.logger.info(\u0027Deleting sharded shard %s/%s\u0027,"},{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_816dbb33","line":577,"in_reply_to":"5f4e5783_c0e2a401","updated":"2017-10-13 10:30:00.000000000","message":"not sure what the question means?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"},{"line_number":581,"context_line":"                    shard_range \u003d broker.get_own_shard_range()"},{"line_number":582,"context_line":"                    shard_range.timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_e0f920c8","line":579,"updated":"2017-10-12 17:29:27.000000000","message":"Wait -- do we not tell the root container about the ranges we found until here? Or is this just one last double-check that it got them?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6408541ab94c9319b20ca19497561df82c7993b2","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"},{"line_number":581,"context_line":"                    shard_range \u003d broker.get_own_shard_range()"},{"line_number":582,"context_line":"                    shard_range.timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_e47c7484","line":579,"in_reply_to":"5f4e5783_21f9674c","updated":"2017-10-13 15:22:19.000000000","message":"So do we need this at all?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"},{"line_number":581,"context_line":"                    shard_range \u003d broker.get_own_shard_range()"},{"line_number":582,"context_line":"                    shard_range.timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_21f9674c","line":579,"in_reply_to":"5f4e5783_e0f920c8","updated":"2017-10-13 10:30:00.000000000","message":"line 569 - if we just scanned and cleaved, we would have updated then. But I feel I may be missing your insight?","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"206f5f4b02a8e054e7fd7ccc8e8710732af77826","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                                     broker.account, broker.container)"},{"line_number":577,"context_line":"                    # We aren\u0027t in the root container."},{"line_number":578,"context_line":"                    self._update_shard_ranges(root_account, root_container,"},{"line_number":579,"context_line":"                                              \u0027PUT\u0027, broker.get_shard_ranges())"},{"line_number":580,"context_line":"                    timestamp \u003d Timestamp.now().internal"},{"line_number":581,"context_line":"                    shard_range \u003d broker.get_own_shard_range()"},{"line_number":582,"context_line":"                    shard_range.timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f4b6375_b8e48fad","line":579,"in_reply_to":"5f4e5783_e47c7484","updated":"2017-10-20 21:36:01.000000000","message":"Ah! Right, now I remember my concern better. 569 calls _update_shard_range_counts which sends *this shard*\u0027s stats. But we never tell the root about the *new shards* we found until here. This worries me a bit...","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                    broker.delete_db(timestamp)"},{"line_number":586,"context_line":"        finally:"},{"line_number":587,"context_line":"            self.logger.increment(\u0027scanned\u0027)"},{"line_number":588,"context_line":"            self.stats[\u0027containers_scanned\u0027] +\u003d 1"},{"line_number":589,"context_line":"            self._periodic_report_stats()"},{"line_number":590,"context_line":""},{"line_number":591,"context_line":"    def _one_shard_cycle(self):"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_c0aaa4d0","line":588,"updated":"2017-10-12 17:29:27.000000000","message":"Right. \"Scanned\" means something different here vs. up above when we\u0027re in the sharding state, but that\u0027s OK.","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9ba6348e51efd78dc564468b7bce0fe8d637fc37","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                    broker.delete_db(timestamp)"},{"line_number":586,"context_line":"        finally:"},{"line_number":587,"context_line":"            self.logger.increment(\u0027scanned\u0027)"},{"line_number":588,"context_line":"            self.stats[\u0027containers_scanned\u0027] +\u003d 1"},{"line_number":589,"context_line":"            self._periodic_report_stats()"},{"line_number":590,"context_line":""},{"line_number":591,"context_line":"    def _one_shard_cycle(self):"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_c16df304","line":588,"in_reply_to":"5f4e5783_c0aaa4d0","updated":"2017-10-13 10:30:00.000000000","message":"at some point I\u0027d like us to audit all logs/stats - scanned here does not mean \u0027looked for shard ranges\u0027 and it may be clearer to change this stat to \u0027processed\u0027 or something other than \u0027scanned\u0027","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a32f8469d0ccb141ea964f73b95cc500fe3cbf87","unresolved":false,"context_lines":[{"line_number":1244,"context_line":"            self.cpool.spawn("},{"line_number":1245,"context_line":"                self._replicate_object, new_part, new_broker.db_file, node_id)"},{"line_number":1246,"context_line":"            self.logger.info(\u0027Node %d sharded %s/%s for shard range %s.\u0027,"},{"line_number":1247,"context_line":"                             node[\u0027id\u0027], broker.account, broker.container,"},{"line_number":1248,"context_line":"                             shard_range)"},{"line_number":1249,"context_line":"            self.logger.increment(\u0027sharded\u0027)"},{"line_number":1250,"context_line":"            self.stats[\u0027containers_sharded\u0027] +\u003d 1"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f4e5783_da56155d","line":1247,"range":{"start_line":1247,"start_character":41,"end_line":1247,"end_character":73},"updated":"2017-10-12 17:29:27.000000000","message":"Off topic: I wonder if it\u0027d be more useful to put broker.db_file in the message instead.","commit_id":"36aefbaa9a2739485279efc0099c7b569429c26f"}]}