)]}'
{"swift/cli/info.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":303,"context_line":"        print(\u0027Sharding Metadata:\u0027)"},{"line_number":304,"context_line":"        shard_type \u003d \u0027root\u0027 if info[\u0027is_root\u0027] else \u0027shard\u0027"},{"line_number":305,"context_line":"        print(\u0027  Type: %s\u0027 % shard_type)"},{"line_number":306,"context_line":"        print(\u0027  State: %s\u0027 % db_state_text(info[\u0027db_state\u0027]))"},{"line_number":307,"context_line":"    if info.get(\u0027shard_ranges\u0027):"},{"line_number":308,"context_line":"        print(\u0027Shard Ranges (%d):\u0027 % len(info[\u0027shard_ranges\u0027]))"},{"line_number":309,"context_line":"        for srange in info[\u0027shard_ranges\u0027]:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0b9c280c","line":306,"updated":"2018-05-11 23:34:37.000000000","message":"Worth doing something with own_shard_range?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        print(\u0027  State: %s\u0027 % db_state_text(info[\u0027db_state\u0027]))"},{"line_number":307,"context_line":"    if info.get(\u0027shard_ranges\u0027):"},{"line_number":308,"context_line":"        print(\u0027Shard Ranges (%d):\u0027 % len(info[\u0027shard_ranges\u0027]))"},{"line_number":309,"context_line":"        for srange in info[\u0027shard_ranges\u0027]:"},{"line_number":310,"context_line":"            srange \u003d dict(srange, state_text\u003dsrange.state_text)"},{"line_number":311,"context_line":"            print(\u0027  Name: %(name)s\u0027 % srange)"},{"line_number":312,"context_line":"            print(\u0027    lower: %(lower)r, upper: %(upper)r\u0027 % srange)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_2ba1e446","line":309,"updated":"2018-05-11 23:34:37.000000000","message":"Only includes live shard ranges, yeah? No deleted?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                     srange[\u0027timestamp\u0027]))"},{"line_number":319,"context_line":"            print(\u0027    Meta Timestamp: %s (%s)\u0027"},{"line_number":320,"context_line":"                  % (Timestamp(srange[\u0027meta_timestamp\u0027]).isoformat,"},{"line_number":321,"context_line":"                     srange[\u0027meta_timestamp\u0027]))"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"def print_obj_metadata(metadata, drop_prefixes\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_cb8950ca","line":321,"updated":"2018-05-11 23:34:37.000000000","message":"Worth including state_timestamp or epoch?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                     srange[\u0027timestamp\u0027]))"},{"line_number":319,"context_line":"            print(\u0027    Meta Timestamp: %s (%s)\u0027"},{"line_number":320,"context_line":"                  % (Timestamp(srange[\u0027meta_timestamp\u0027]).isoformat,"},{"line_number":321,"context_line":"                     srange[\u0027meta_timestamp\u0027]))"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"def print_obj_metadata(metadata, drop_prefixes\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_9d2290b8","line":321,"in_reply_to":"5f7c97a3_6fe348a5","updated":"2018-05-15 17:46:09.000000000","message":"https://trello.com/c/yAGd28os","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                     srange[\u0027timestamp\u0027]))"},{"line_number":319,"context_line":"            print(\u0027    Meta Timestamp: %s (%s)\u0027"},{"line_number":320,"context_line":"                  % (Timestamp(srange[\u0027meta_timestamp\u0027]).isoformat,"},{"line_number":321,"context_line":"                     srange[\u0027meta_timestamp\u0027]))"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"def print_obj_metadata(metadata, drop_prefixes\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_6fe348a5","line":321,"in_reply_to":"5f7c97a3_79dfa839","updated":"2018-05-14 23:32:48.000000000","message":"\u003e With many shards there will be a lot of output...\n\nDefinitely noticed that in practice. An arg to control whether to display all shard ranges or just a summary (and defaulting to the summary) would definitely be nice.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                     srange[\u0027timestamp\u0027]))"},{"line_number":319,"context_line":"            print(\u0027    Meta Timestamp: %s (%s)\u0027"},{"line_number":320,"context_line":"                  % (Timestamp(srange[\u0027meta_timestamp\u0027]).isoformat,"},{"line_number":321,"context_line":"                     srange[\u0027meta_timestamp\u0027]))"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"def print_obj_metadata(metadata, drop_prefixes\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_79dfa839","line":321,"in_reply_to":"5f7c97a3_cb8950ca","updated":"2018-05-14 10:51:03.000000000","message":"All good points but I\u0027m starting to wonder if we should display all the shard range details here or defer to \u0027swift-manage-shard-ranges info\u0027. Or vice-versa. I\u0027m concious I have already added own shard range to swift-manage-shard-ranges. \n\nWith many shards there will be a lot of output...hmmm, maybe I\u0027ll add a CLI arg to control detail level.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"}],"swift/common/db.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":1006,"context_line":"            \u0027 WHERE status_changed_at \u003c ?\u0027 % self.db_type,"},{"line_number":1007,"context_line":"            (timestamp, timestamp))"},{"line_number":1008,"context_line":""},{"line_number":1009,"context_line":"    def get_brokers(self):"},{"line_number":1010,"context_line":"        return [self]"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_92539617","line":1010,"range":{"start_line":1009,"start_character":4,"end_line":1010,"end_character":21},"updated":"2018-05-14 10:51:03.000000000","message":"not needed","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1006,"context_line":"            \u0027 WHERE status_changed_at \u003c ?\u0027 % self.db_type,"},{"line_number":1007,"context_line":"            (timestamp, timestamp))"},{"line_number":1008,"context_line":""},{"line_number":1009,"context_line":"    def get_brokers(self):"},{"line_number":1010,"context_line":"        return [self]"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_af972048","line":1010,"range":{"start_line":1009,"start_character":4,"end_line":1010,"end_character":21},"in_reply_to":"5f7c97a3_92539617","updated":"2018-05-14 23:32:48.000000000","message":"(yet)\n\n...right?","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1006,"context_line":"            \u0027 WHERE status_changed_at \u003c ?\u0027 % self.db_type,"},{"line_number":1007,"context_line":"            (timestamp, timestamp))"},{"line_number":1008,"context_line":""},{"line_number":1009,"context_line":"    def get_brokers(self):"},{"line_number":1010,"context_line":"        return [self]"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_b820ba9c","line":1010,"range":{"start_line":1009,"start_character":4,"end_line":1010,"end_character":21},"in_reply_to":"5f7c97a3_af972048","updated":"2018-05-15 17:46:09.000000000","message":"it wasn\u0027t needed in DatabaseBroker, only in ContainerBroker","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"}],"swift/common/db_replicator.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":967,"context_line":"            objects \u003d existing_broker.get_items_since(point, 1000)"},{"line_number":968,"context_line":"            sleep()"},{"line_number":969,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":970,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":971,"context_line":"        # limit in the future."},{"line_number":972,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"},{"line_number":973,"context_line":"        if other_items:"},{"line_number":974,"context_line":"            new_broker.merge_items(other_items)"},{"line_number":975,"context_line":"        new_broker.newid(args[0])"},{"line_number":976,"context_line":"        new_broker.update_metadata(existing_broker.metadata)"},{"line_number":977,"context_line":"        if self._abort_rsync_then_merge(db_file, old_filename):"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_7dea4b0f","line":974,"range":{"start_line":970,"start_character":9,"end_line":974,"end_character":47},"updated":"2018-05-14 10:51:03.000000000","message":"we should use a hook method for subclasses to override","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":121,"context_line":"                    if not os.path.isdir(hash_dir):"},{"line_number":122,"context_line":"                        continue"},{"line_number":123,"context_line":"                    object_file \u003d os.path.join(hash_dir, hsh + \u0027.db\u0027)"},{"line_number":124,"context_line":"                    if os.path.exists(object_file):"},{"line_number":125,"context_line":"                        yield (partition, object_file, context)"},{"line_number":126,"context_line":"                        continue"},{"line_number":127,"context_line":"                    # look for any alternate db filenames"},{"line_number":128,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":129,"context_line":"                    if db_files:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_ab1bb4c7","line":126,"range":{"start_line":124,"start_character":20,"end_line":126,"end_character":32},"updated":"2018-05-11 23:34:37.000000000","message":"Do we still need this? If it\u0027s just the hash.db, get_db_files will still find it...\n\nI guess this makes a difference when we\u0027re in the process of sharding? So we\u0027ve got both the hash.db and a hash_epoch.db... and we only want to deal with the hash.db? I\u0027m not sure that\u0027s right.\n\nAnd what happens when we\u0027ve shrunk back down so the hash_epoch.db is in charge of object rows again -- but then it gets big so we have a hash_epoch2.db and start sharding hash_epoch.db -- I think we\u0027d start working with hash_epoch2.db *immediately*! Seems weird to have different behavior the second time around.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":121,"context_line":"                    if not os.path.isdir(hash_dir):"},{"line_number":122,"context_line":"                        continue"},{"line_number":123,"context_line":"                    object_file \u003d os.path.join(hash_dir, hsh + \u0027.db\u0027)"},{"line_number":124,"context_line":"                    if os.path.exists(object_file):"},{"line_number":125,"context_line":"                        yield (partition, object_file, context)"},{"line_number":126,"context_line":"                        continue"},{"line_number":127,"context_line":"                    # look for any alternate db filenames"},{"line_number":128,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":129,"context_line":"                    if db_files:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_652272bb","line":126,"range":{"start_line":124,"start_character":20,"end_line":126,"end_character":32},"in_reply_to":"5f7c97a3_5932ec5f","updated":"2018-05-15 23:27:29.000000000","message":"Makes me think we really ought to consider just handing out hash_dir and let ContainerBroker go from there. Still have ContainerBroker support a direct file, though, so we can work in tmp_dir when making the preshard db (and to not needlessly break anything that may have been built assuming that ContainerBroker should take a file).","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":121,"context_line":"                    if not os.path.isdir(hash_dir):"},{"line_number":122,"context_line":"                        continue"},{"line_number":123,"context_line":"                    object_file \u003d os.path.join(hash_dir, hsh + \u0027.db\u0027)"},{"line_number":124,"context_line":"                    if os.path.exists(object_file):"},{"line_number":125,"context_line":"                        yield (partition, object_file, context)"},{"line_number":126,"context_line":"                        continue"},{"line_number":127,"context_line":"                    # look for any alternate db filenames"},{"line_number":128,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":129,"context_line":"                    if db_files:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_3eeb1678","line":126,"range":{"start_line":124,"start_character":20,"end_line":126,"end_character":32},"in_reply_to":"5f7c97a3_652272bb","updated":"2018-05-16 18:49:30.000000000","message":"I\u0027m going to leave this as it is for now, seems like it can be changed later on master.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":121,"context_line":"                    if not os.path.isdir(hash_dir):"},{"line_number":122,"context_line":"                        continue"},{"line_number":123,"context_line":"                    object_file \u003d os.path.join(hash_dir, hsh + \u0027.db\u0027)"},{"line_number":124,"context_line":"                    if os.path.exists(object_file):"},{"line_number":125,"context_line":"                        yield (partition, object_file, context)"},{"line_number":126,"context_line":"                        continue"},{"line_number":127,"context_line":"                    # look for any alternate db filenames"},{"line_number":128,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":129,"context_line":"                    if db_files:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_5932ec5f","line":126,"range":{"start_line":124,"start_character":20,"end_line":126,"end_character":32},"in_reply_to":"5f7c97a3_ab1bb4c7","updated":"2018-05-14 10:51:03.000000000","message":"IIRC perhaps this was just avoiding a listdir in the common case. It doesn\u0027t actually matter which db file is passed to the ContainerBroker - it lists the dir to find the \u0027newest\u0027. Maybe it is an inconsequential efficiency.\n\nI agree that it looks unnecessary.\n\nWhat do you think - leave, add comment, or remove?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":938,"context_line":"    def complete_rsync(self, drive, db_file, args):"},{"line_number":939,"context_line":"        old_filename \u003d os.path.join(self.root, drive, \u0027tmp\u0027, args[0])"},{"line_number":940,"context_line":"        if args[1:]:"},{"line_number":941,"context_line":"            db_file \u003d os.path.join(os.path.dirname(db_file), args[1])"},{"line_number":942,"context_line":"        if os.path.exists(db_file):"},{"line_number":943,"context_line":"            return HTTPNotFound()"},{"line_number":944,"context_line":"        if not os.path.exists(old_filename):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_4b7a20f1","line":941,"updated":"2018-05-11 23:34:37.000000000","message":"+1, nice rolling-upgrade story","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":969,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":970,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":971,"context_line":"        # limit in the future."},{"line_number":972,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"},{"line_number":973,"context_line":"        if other_items:"},{"line_number":974,"context_line":"            new_broker.merge_items(other_items)"},{"line_number":975,"context_line":"        new_broker.newid(args[0])"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0b80a8be","line":972,"range":{"start_line":972,"start_character":38,"end_line":972,"end_character":65},"updated":"2018-05-11 23:34:37.000000000","message":"So abstract dbs only use this method in rsync-then-merge? Hmm...\n\nAh -- going away following https://review.openstack.org/#/c/567920/ ?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":969,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":970,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":971,"context_line":"        # limit in the future."},{"line_number":972,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"},{"line_number":973,"context_line":"        if other_items:"},{"line_number":974,"context_line":"            new_broker.merge_items(other_items)"},{"line_number":975,"context_line":"        new_broker.newid(args[0])"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_39261017","line":972,"range":{"start_line":972,"start_character":38,"end_line":972,"end_character":65},"in_reply_to":"5f7c97a3_0b80a8be","updated":"2018-05-14 10:51:03.000000000","message":"yes, getting rid of specifics in 567920","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":321,"context_line":"        with Timeout(replicate_timeout or self.node_timeout):"},{"line_number":322,"context_line":"            response \u003d http.replicate(replicate_method, local_id,"},{"line_number":323,"context_line":"                                      os.path.basename(broker.db_file))"},{"line_number":324,"context_line":"        return response and 200 \u003c\u003d response.status \u003c 300"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"    def _send_merge_items(self, http, op, local_id, items):"},{"line_number":327,"context_line":"        with Timeout(self.node_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_8f534439","line":324,"updated":"2018-05-14 23:32:48.000000000","message":"Off-topic: do we ever log errors here?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":323,"context_line":"                                      os.path.basename(broker.db_file))"},{"line_number":324,"context_line":"        return response and 200 \u003c\u003d response.status \u003c 300"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"    def _send_merge_items(self, http, op, local_id, items):"},{"line_number":327,"context_line":"        with Timeout(self.node_timeout):"},{"line_number":328,"context_line":"            response \u003d http.replicate(op, items, local_id)"},{"line_number":329,"context_line":"        if not response or not is_success(response.status):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_afce4060","line":326,"range":{"start_line":326,"start_character":8,"end_line":326,"end_character":25},"updated":"2018-05-14 23:32:48.000000000","message":"nit: This got more general -- maybe worth renaming?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":323,"context_line":"                                      os.path.basename(broker.db_file))"},{"line_number":324,"context_line":"        return response and 200 \u003c\u003d response.status \u003c 300"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"    def _send_merge_items(self, http, op, local_id, items):"},{"line_number":327,"context_line":"        with Timeout(self.node_timeout):"},{"line_number":328,"context_line":"            response \u003d http.replicate(op, items, local_id)"},{"line_number":329,"context_line":"        if not response or not is_success(response.status):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_783ea2c1","line":326,"range":{"start_line":326,"start_character":8,"end_line":326,"end_character":25},"in_reply_to":"5f7c97a3_afce4060","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        while len(objects) and diffs \u003c self.max_diffs:"},{"line_number":359,"context_line":"            diffs +\u003d 1"},{"line_number":360,"context_line":"            if not self._send_merge_items("},{"line_number":361,"context_line":"                    http, \u0027merge_items\u0027, local_id, objects):"},{"line_number":362,"context_line":"                return False"},{"line_number":363,"context_line":"            # replication relies on db order to send the next merge batch in"},{"line_number":364,"context_line":"            # order with no gaps"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_8f010409","line":361,"updated":"2018-05-14 23:32:48.000000000","message":"... \u0027cause this feels pretty redundant now.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        while len(objects) and diffs \u003c self.max_diffs:"},{"line_number":359,"context_line":"            diffs +\u003d 1"},{"line_number":360,"context_line":"            if not self._send_merge_items("},{"line_number":361,"context_line":"                    http, \u0027merge_items\u0027, local_id, objects):"},{"line_number":362,"context_line":"                return False"},{"line_number":363,"context_line":"            # replication relies on db order to send the next merge batch in"},{"line_number":364,"context_line":"            # order with no gaps"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_585de612","line":361,"in_reply_to":"5f7c97a3_8f010409","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":421,"context_line":""},{"line_number":422,"context_line":"        :returns: ReplConnection object"},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        hsh, other, ext \u003d parse_db_filename(os.path.basename(db_file))"},{"line_number":425,"context_line":"        return ReplConnection(node, partition, hsh, self.logger)"},{"line_number":426,"context_line":""},{"line_number":427,"context_line":"    def _gather_sync_args(self, info):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_ea10a650","line":424,"range":{"start_line":424,"start_character":44,"end_line":424,"end_character":69},"updated":"2018-05-14 23:32:48.000000000","message":"nit: Can just be db_file -- first thing parse_db_filename does is get the basename anyway.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":421,"context_line":""},{"line_number":422,"context_line":"        :returns: ReplConnection object"},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        hsh, other, ext \u003d parse_db_filename(os.path.basename(db_file))"},{"line_number":425,"context_line":"        return ReplConnection(node, partition, hsh, self.logger)"},{"line_number":426,"context_line":""},{"line_number":427,"context_line":"    def _gather_sync_args(self, info):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_d829f636","line":424,"range":{"start_line":424,"start_character":44,"end_line":424,"end_character":69},"in_reply_to":"5f7c97a3_ea10a650","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":321,"context_line":"        with Timeout(replicate_timeout or self.node_timeout):"},{"line_number":322,"context_line":"            response \u003d http.replicate(replicate_method, local_id,"},{"line_number":323,"context_line":"                                      os.path.basename(broker.db_file))"},{"line_number":324,"context_line":"        return response and 200 \u003c\u003d response.status \u003c 300"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"    def _send_replicate_request(self, http, *repl_args):"},{"line_number":327,"context_line":"        with Timeout(self.node_timeout):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_5026116a","line":324,"updated":"2018-05-15 23:27:29.000000000","message":"So close to being able to refactor this as\n\n return self._send_replicate_request(\n     http, replicate_method, local_id,\n     os.path.basename(broker.db_file))\n\nLousy replicate_timeout...","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        while len(objects) and diffs \u003c self.max_diffs:"},{"line_number":359,"context_line":"            diffs +\u003d 1"},{"line_number":360,"context_line":"            if not self._send_replicate_request("},{"line_number":361,"context_line":"                    http, \u0027merge_items\u0027, objects, local_id):"},{"line_number":362,"context_line":"                return False"},{"line_number":363,"context_line":"            # replication relies on db order to send the next merge batch in"},{"line_number":364,"context_line":"            # order with no gaps"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_b0a685c0","line":361,"range":{"start_line":361,"start_character":41,"end_line":361,"end_character":58},"updated":"2018-05-15 23:27:29.000000000","message":"Yeah, it\u0027s kinda weird that _send_merge_items swaps these in https://review.openstack.org/#/c/565742\n\nNot sure whether it\u0027s worth fixing (or doing the rename/signature change) in that first patch, though.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        while len(objects) and diffs \u003c self.max_diffs:"},{"line_number":359,"context_line":"            diffs +\u003d 1"},{"line_number":360,"context_line":"            if not self._send_replicate_request("},{"line_number":361,"context_line":"                    http, \u0027merge_items\u0027, objects, local_id):"},{"line_number":362,"context_line":"                return False"},{"line_number":363,"context_line":"            # replication relies on db order to send the next merge batch in"},{"line_number":364,"context_line":"            # order with no gaps"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_e142df3d","line":361,"range":{"start_line":361,"start_character":41,"end_line":361,"end_character":58},"in_reply_to":"5f7c97a3_b0a685c0","updated":"2018-05-16 18:49:30.000000000","message":"hmmm, yeah I did that in wrong patch but will leave as is now since the ancestor patch seems stable with +2s","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"}],"swift/common/utils.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5291,"context_line":"    return out"},{"line_number":5292,"context_line":""},{"line_number":5293,"context_line":""},{"line_number":5294,"context_line":"# TODO: unit test"},{"line_number":5295,"context_line":"def parse_db_filename(filename):"},{"line_number":5296,"context_line":"    \"\"\""},{"line_number":5297,"context_line":"    Splits a db filename into three parts: the hash, the epoch, and the"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_b247b24c","line":5294,"range":{"start_line":5294,"start_character":0,"end_line":5294,"end_character":17},"updated":"2018-05-14 10:51:03.000000000","message":"done - incorporated some of the paste test","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5291,"context_line":"    return out"},{"line_number":5292,"context_line":""},{"line_number":5293,"context_line":""},{"line_number":5294,"context_line":"# TODO: unit test"},{"line_number":5295,"context_line":"def parse_db_filename(filename):"},{"line_number":5296,"context_line":"    \"\"\""},{"line_number":5297,"context_line":"    Splits a db filename into three parts: the hash, the epoch, and the"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_687fe2df","line":5294,"updated":"2018-05-11 23:34:37.000000000","message":"http://paste.openstack.org/show/720848/ -- oh, I was looking at an old patchset...","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5313,"context_line":"    return hash_, epoch, ext"},{"line_number":5314,"context_line":""},{"line_number":5315,"context_line":""},{"line_number":5316,"context_line":"# TODO: unit test"},{"line_number":5317,"context_line":"def make_db_file_path(db_path, epoch):"},{"line_number":5318,"context_line":"    \"\"\""},{"line_number":5319,"context_line":"    Given a path to a db file, return a modified path whose filename part has"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_32a2c2fd","line":5316,"range":{"start_line":5316,"start_character":0,"end_line":5316,"end_character":17},"updated":"2018-05-14 10:51:03.000000000","message":"done - incorporated some of the paste test","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5313,"context_line":"    return hash_, epoch, ext"},{"line_number":5314,"context_line":""},{"line_number":5315,"context_line":""},{"line_number":5316,"context_line":"# TODO: unit test"},{"line_number":5317,"context_line":"def make_db_file_path(db_path, epoch):"},{"line_number":5318,"context_line":"    \"\"\""},{"line_number":5319,"context_line":"    Given a path to a db file, return a modified path whose filename part has"}],"source_content_type":"text/x-python","patch_set":5,"id":"5f7c97a3_6b3d9c8d","line":5316,"updated":"2018-05-11 23:34:37.000000000","message":"http://paste.openstack.org/show/720849/","commit_id":"e58fdc6269323a4102aa90f2563d306d4d24c295"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":4894,"context_line":"            params[\u0027state_timestamp\u0027], params[\u0027epoch\u0027])"},{"line_number":4895,"context_line":""},{"line_number":4896,"context_line":""},{"line_number":4897,"context_line":"def find_shard_range(item, ranges):"},{"line_number":4898,"context_line":"    \"\"\""},{"line_number":4899,"context_line":""},{"line_number":4900,"context_line":"    :param item: The item that needs to be placed"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0b59888c","line":4897,"updated":"2018-05-11 23:34:37.000000000","message":"This might be better in the parent patch, since it has some pretty big implications for ShardRange.__lt__ and ShardRange.__gt__","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":4894,"context_line":"            params[\u0027state_timestamp\u0027], params[\u0027epoch\u0027])"},{"line_number":4895,"context_line":""},{"line_number":4896,"context_line":""},{"line_number":4897,"context_line":"def find_shard_range(item, ranges):"},{"line_number":4898,"context_line":"    \"\"\""},{"line_number":4899,"context_line":""},{"line_number":4900,"context_line":"    :param item: The item that needs to be placed"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_39d930f2","line":4897,"in_reply_to":"5f7c97a3_0b59888c","updated":"2018-05-14 10:51:03.000000000","message":"+1","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5303,"context_line":"    :param filename: A db file basename or path to a db file."},{"line_number":5304,"context_line":"    :return: A tuple of (hash , epoch, extension). ``epoch`` may be None."},{"line_number":5305,"context_line":"    \"\"\""},{"line_number":5306,"context_line":"    filename \u003d os.path.basename(filename)"},{"line_number":5307,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":5308,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":5309,"context_line":"    hash_ \u003d parts.pop(0)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_cb56305d","line":5306,"updated":"2018-05-11 23:34:37.000000000","message":"So what happens when we pass this a directory? Like\n\n /path/to/dir/\n\n?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5303,"context_line":"    :param filename: A db file basename or path to a db file."},{"line_number":5304,"context_line":"    :return: A tuple of (hash , epoch, extension). ``epoch`` may be None."},{"line_number":5305,"context_line":"    \"\"\""},{"line_number":5306,"context_line":"    filename \u003d os.path.basename(filename)"},{"line_number":5307,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":5308,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":5309,"context_line":"    hash_ \u003d parts.pop(0)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_6f803647","line":5306,"in_reply_to":"5f7c97a3_cb56305d","updated":"2018-05-14 10:51:03.000000000","message":"The docstring is pretty clear about expecting a filename, so I\u0027ll add a ValueError if a dir is passed in.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5307,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":5308,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":5309,"context_line":"    hash_ \u003d parts.pop(0)"},{"line_number":5310,"context_line":"    epoch \u003d parts[0] if parts else None"},{"line_number":5311,"context_line":"    return hash_, epoch, ext"},{"line_number":5312,"context_line":""},{"line_number":5313,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_8b4cb8c8","line":5310,"range":{"start_line":5310,"start_character":12,"end_line":5310,"end_character":20},"updated":"2018-05-11 23:34:37.000000000","message":"What if the DB\u0027s epoch includes an offset?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5307,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":5308,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":5309,"context_line":"    hash_ \u003d parts.pop(0)"},{"line_number":5310,"context_line":"    epoch \u003d parts[0] if parts else None"},{"line_number":5311,"context_line":"    return hash_, epoch, ext"},{"line_number":5312,"context_line":""},{"line_number":5313,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_efb20649","line":5310,"range":{"start_line":5310,"start_character":12,"end_line":5310,"end_character":20},"in_reply_to":"5f7c97a3_8b4cb8c8","updated":"2018-05-14 10:51:03.000000000","message":"Epochs don\u0027t have offsets.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5308,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":5309,"context_line":"    hash_ \u003d parts.pop(0)"},{"line_number":5310,"context_line":"    epoch \u003d parts[0] if parts else None"},{"line_number":5311,"context_line":"    return hash_, epoch, ext"},{"line_number":5312,"context_line":""},{"line_number":5313,"context_line":""},{"line_number":5314,"context_line":"def make_db_file_path(db_path, epoch):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_ab4974b7","line":5311,"updated":"2018-05-11 23:34:37.000000000","message":"Alternatively,\n\n hash_, epoch \u003d name.partition(\u0027_\u0027)[::2]\n return hash_, epoch or None, ext\n\n(Though it changes the result for some weird names that should never come up in practice.)\n\n*shrug*","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5327,"context_line":"    :raises ValueError: if the ``epoch`` is not valid for constructing a"},{"line_number":5328,"context_line":"        :class:`~swift.common.utils.Timestamp`."},{"line_number":5329,"context_line":"    \"\"\""},{"line_number":5330,"context_line":"    epoch \u003d Timestamp(epoch).normal"},{"line_number":5331,"context_line":"    hash_, _, ext \u003d parse_db_filename(db_path)"},{"line_number":5332,"context_line":"    db_dir \u003d os.path.dirname(db_path)"},{"line_number":5333,"context_line":"    return os.path.join(db_dir, \u0027%s_%s%s\u0027 % (hash_, epoch, ext))"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0bce682c","line":5330,"range":{"start_line":5330,"start_character":29,"end_line":5330,"end_character":35},"updated":"2018-05-11 23:34:37.000000000","message":"Ah -- we truncate any offset. I guess that\u0027s probably OK...","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5343,"context_line":"        ``db_path``. This list may be empty."},{"line_number":5344,"context_line":"    \"\"\""},{"line_number":5345,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":5346,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":5347,"context_line":"        return []"},{"line_number":5348,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":5349,"context_line":"    if not files:"},{"line_number":5350,"context_line":"        return []"},{"line_number":5351,"context_line":"    match_hash, epoch, ext \u003d parse_db_filename(db_file)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_cbdb90ea","line":5348,"range":{"start_line":5346,"start_character":4,"end_line":5348,"end_character":30},"updated":"2018-05-11 23:34:37.000000000","message":"Could probably save a stat() with\n\n try:\n     files \u003d os.listdir(db_dir)\n except OSError as err:\n     if err.errno \u003d\u003d errno.ENOENT:\n         return []\n     raise","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5343,"context_line":"        ``db_path``. This list may be empty."},{"line_number":5344,"context_line":"    \"\"\""},{"line_number":5345,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":5346,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":5347,"context_line":"        return []"},{"line_number":5348,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":5349,"context_line":"    if not files:"},{"line_number":5350,"context_line":"        return []"},{"line_number":5351,"context_line":"    match_hash, epoch, ext \u003d parse_db_filename(db_file)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_6fb836e4","line":5348,"range":{"start_line":5346,"start_character":4,"end_line":5348,"end_character":30},"in_reply_to":"5f7c97a3_cbdb90ea","updated":"2018-05-14 10:51:03.000000000","message":"Done","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5348,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":5349,"context_line":"    if not files:"},{"line_number":5350,"context_line":"        return []"},{"line_number":5351,"context_line":"    match_hash, epoch, ext \u003d parse_db_filename(db_file)"},{"line_number":5352,"context_line":"    results \u003d []"},{"line_number":5353,"context_line":"    for f in files:"},{"line_number":5354,"context_line":"        hash_, epoch, ext \u003d parse_db_filename(f)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_ebd8ccef","line":5351,"range":{"start_line":5351,"start_character":23,"end_line":5351,"end_character":26},"updated":"2018-05-11 23:34:37.000000000","message":"We never actually check that ext is \u0027.db\u0027... so you could have a case where db_path *exists* but get_db_files(db_path) returns an empty array... *shrug*","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":5348,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":5349,"context_line":"    if not files:"},{"line_number":5350,"context_line":"        return []"},{"line_number":5351,"context_line":"    match_hash, epoch, ext \u003d parse_db_filename(db_file)"},{"line_number":5352,"context_line":"    results \u003d []"},{"line_number":5353,"context_line":"    for f in files:"},{"line_number":5354,"context_line":"        hash_, epoch, ext \u003d parse_db_filename(f)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_b63ea1be","line":5351,"range":{"start_line":5351,"start_character":23,"end_line":5351,"end_character":26},"in_reply_to":"5f7c97a3_0f7f4215","updated":"2018-05-14 23:32:48.000000000","message":"Interesting, particularly coupled with the light shed by your comment at https://review.openstack.org/#/c/565742/4/swift/common/db_replicator.py@953 ...\n\nShould we use\n\n match_hash \u003d os.path.basename(db_dir)\n\ninstead of parsing db_file? Then I think we could just pass the directory to db_path...","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":5348,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":5349,"context_line":"    if not files:"},{"line_number":5350,"context_line":"        return []"},{"line_number":5351,"context_line":"    match_hash, epoch, ext \u003d parse_db_filename(db_file)"},{"line_number":5352,"context_line":"    results \u003d []"},{"line_number":5353,"context_line":"    for f in files:"},{"line_number":5354,"context_line":"        hash_, epoch, ext \u003d parse_db_filename(f)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0f7f4215","line":5351,"range":{"start_line":5351,"start_character":23,"end_line":5351,"end_character":26},"in_reply_to":"5f7c97a3_ebd8ccef","updated":"2018-05-14 10:51:03.000000000","message":"...which would be correct, since method only returns *valid* db files.\n\nI\u0027ll add some clarification of \u0027valid\u0027 to the docstring","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":5357,"context_line":"        if hash_ !\u003d match_hash:"},{"line_number":5358,"context_line":"            continue"},{"line_number":5359,"context_line":"        results.append(os.path.join(db_dir, f))"},{"line_number":5360,"context_line":"    return sorted(results)"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_4bf1c06a","line":5360,"updated":"2018-05-11 23:34:37.000000000","message":"Good thing\n\n (\u0027.\u0027 \u003c \u0027_\u0027) \u003d\u003d True","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":5328,"context_line":"        :class:`~swift.common.utils.Timestamp`."},{"line_number":5329,"context_line":"    \"\"\""},{"line_number":5330,"context_line":"    if epoch is None:"},{"line_number":5331,"context_line":"        raise ValueError(\u0027epoch must not be None\u0027)"},{"line_number":5332,"context_line":"    epoch \u003d Timestamp(epoch).normal"},{"line_number":5333,"context_line":"    hash_, _, ext \u003d parse_db_filename(db_path)"},{"line_number":5334,"context_line":"    db_dir \u003d os.path.dirname(db_path)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_aa670eb7","line":5331,"updated":"2018-05-14 23:32:48.000000000","message":"I wonder if it might be useful to have epoch\u003dNone always get you back to \u003chash\u003e.db...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":5328,"context_line":"        :class:`~swift.common.utils.Timestamp`."},{"line_number":5329,"context_line":"    \"\"\""},{"line_number":5330,"context_line":"    if epoch is None:"},{"line_number":5331,"context_line":"        raise ValueError(\u0027epoch must not be None\u0027)"},{"line_number":5332,"context_line":"    epoch \u003d Timestamp(epoch).normal"},{"line_number":5333,"context_line":"    hash_, _, ext \u003d parse_db_filename(db_path)"},{"line_number":5334,"context_line":"    db_dir \u003d os.path.dirname(db_path)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_b8dbda4c","line":5331,"in_reply_to":"5f7c97a3_aa670eb7","updated":"2018-05-15 17:46:09.000000000","message":"maybe, didn\u0027t get round to exploring this today. maybe follow-up on master?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":38,"context_line":"DATADIR \u003d \u0027containers\u0027"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"RECORD_TYPE_OBJECT \u003d \u0027object\u0027"},{"line_number":41,"context_line":"RECORD_TYPE_SHARD_NODE \u003d \u0027shard\u0027"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"NOTFOUND \u003d 0"},{"line_number":44,"context_line":"UNSHARDED \u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_0be748ae","line":41,"range":{"start_line":41,"start_character":17,"end_line":41,"end_character":22},"updated":"2018-05-11 23:34:37.000000000","message":"Maybe better as just\n\n RECORD_TYPE_SHARD\n\n?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":38,"context_line":"DATADIR \u003d \u0027containers\u0027"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"RECORD_TYPE_OBJECT \u003d \u0027object\u0027"},{"line_number":41,"context_line":"RECORD_TYPE_SHARD_NODE \u003d \u0027shard\u0027"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"NOTFOUND \u003d 0"},{"line_number":44,"context_line":"UNSHARDED \u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_4a06881e","line":41,"range":{"start_line":41,"start_character":17,"end_line":41,"end_character":22},"in_reply_to":"5f7c97a3_0be748ae","updated":"2018-05-14 10:51:03.000000000","message":"+1 no idea what NODE is doing there","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":45,"context_line":"SHARDING \u003d 2"},{"line_number":46,"context_line":"SHARDED \u003d 3"},{"line_number":47,"context_line":"COLLAPSED \u003d 4"},{"line_number":48,"context_line":"DB_STATES \u003d [\u0027not_found\u0027, \u0027unsharded\u0027, \u0027sharding\u0027, \u0027sharded\u0027, \u0027collapsed\u0027]"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"def db_state_text(state):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_cb04f04c","line":48,"updated":"2018-05-11 23:34:37.000000000","message":"Remind me why we want the integer values instead of just relying on string constants.\n\nI *think* https://review.openstack.org/567961 still works?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":279,"context_line":""},{"line_number":280,"context_line":"    new_content \u003d False"},{"line_number":281,"context_line":"    # timestamp must be the same, so preserve existing range bounds"},{"line_number":282,"context_line":"    for k in (\u0027lower\u0027, \u0027upper\u0027):"},{"line_number":283,"context_line":"        shard_data[k] \u003d existing[k]"},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"    # now we need to look for meta data updates"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_eb01ac39","line":282,"updated":"2018-05-11 23:34:37.000000000","message":"Should we preserve deleted as well? Changing that should also change timestamp, right?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":279,"context_line":""},{"line_number":280,"context_line":"    new_content \u003d False"},{"line_number":281,"context_line":"    # timestamp must be the same, so preserve existing range bounds"},{"line_number":282,"context_line":"    for k in (\u0027lower\u0027, \u0027upper\u0027):"},{"line_number":283,"context_line":"        shard_data[k] \u003d existing[k]"},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"    # now we need to look for meta data updates"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_8aea600c","line":282,"in_reply_to":"5f7c97a3_eb01ac39","updated":"2018-05-14 10:51:03.000000000","message":"good catch, that\u0027s a bug. must add a test for that too.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":297,"context_line":"            shard_data[k] \u003d existing[k]"},{"line_number":298,"context_line":"    else:"},{"line_number":299,"context_line":"        new_content \u003d True"},{"line_number":300,"context_line":"    return new_content"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"class ContainerBroker(DatabaseBroker):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_cb0e303a","line":300,"updated":"2018-05-11 23:34:37.000000000","message":"OK, so we\u0027ve got three classes of data:\n\n- data tied to timestamp (lower, upper, deleted) -- and if timestamp changes, we\u0027re in a whole new world and everything gets blown away,\n- data tied to meta_timestamp (object_count, bytes_used), and\n- data tied to state_timestamp (state, epoch).\n\nThose latter two advance independently -- given two shards with the same timestamp, but\n\n- A has meta_timestamp t1 and state_timestamp t4 and\n- B has meta_timestamp t3 and state_timestamp t2\n\nwe want to save B\u0027s stats and A\u0027s state/epoch -- yeah?","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":297,"context_line":"            shard_data[k] \u003d existing[k]"},{"line_number":298,"context_line":"    else:"},{"line_number":299,"context_line":"        new_content \u003d True"},{"line_number":300,"context_line":"    return new_content"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"class ContainerBroker(DatabaseBroker):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_8a902045","line":300,"in_reply_to":"5f7c97a3_cb0e303a","updated":"2018-05-14 10:51:03.000000000","message":"correct","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"104ecb99be3f691c8095e1771d28204e9313eaa8","unresolved":false,"context_lines":[{"line_number":1790,"context_line":"    def get_shard_usage(self):"},{"line_number":1791,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dSHARD_STATS_STATES)"},{"line_number":1792,"context_line":"        return {\u0027bytes_used\u0027: sum([sr.bytes_used for sr in shard_ranges]),"},{"line_number":1793,"context_line":"                \u0027object_count\u0027: sum([sr.object_count for sr in shard_ranges])}"},{"line_number":1794,"context_line":""},{"line_number":1795,"context_line":"    def get_other_replication_items(self):"},{"line_number":1796,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_deleted\u003dTrue,"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_4b96a0f2","line":1793,"updated":"2018-05-11 23:34:37.000000000","message":"nit: Don\u0027t need the list;\n\n {\u0027bytes_used\u0027: sum(sr.bytes_used for sr in shard_ranges),\n  \u0027object_count\u0027: sum(sr.object_count for sr in shard_ranges)}\n\nis fine, and saves some small amount of memory.","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8f5dbff7a8c1617401beb6b7e0daa3b1173535af","unresolved":false,"context_lines":[{"line_number":1790,"context_line":"    def get_shard_usage(self):"},{"line_number":1791,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dSHARD_STATS_STATES)"},{"line_number":1792,"context_line":"        return {\u0027bytes_used\u0027: sum([sr.bytes_used for sr in shard_ranges]),"},{"line_number":1793,"context_line":"                \u0027object_count\u0027: sum([sr.object_count for sr in shard_ranges])}"},{"line_number":1794,"context_line":""},{"line_number":1795,"context_line":"    def get_other_replication_items(self):"},{"line_number":1796,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_deleted\u003dTrue,"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_2aa1cc94","line":1793,"in_reply_to":"5f7c97a3_4b96a0f2","updated":"2018-05-14 10:51:03.000000000","message":"Done","commit_id":"02a1aad986d878c4fd0c65bc81ffd4675cd9677c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":313,"context_line":"        sharded."},{"line_number":314,"context_line":"      * :attr:`db_files` is a list of existing db files for the broker. This"},{"line_number":315,"context_line":"        list should have at least one entry for an initialised broker, and will"},{"line_number":316,"context_line":"        have two entries while a broker is in SHARDING state."},{"line_number":317,"context_line":"      * :attr:`db_file` is the path to whichever db is currently authoritative"},{"line_number":318,"context_line":"        for the container. Depending on the container\u0027s state, this may not be"},{"line_number":319,"context_line":"        the same as the ``db_file`` argument given to :meth:`~__init__`, unless"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_6ad8167a","line":316,"range":{"start_line":316,"start_character":13,"end_line":316,"end_character":24},"updated":"2018-05-14 23:32:48.000000000","message":"Always exactly two? Seems like we\u0027re trying to allow for more (just in case) but I\u0027m not sure it should come up...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":313,"context_line":"        sharded."},{"line_number":314,"context_line":"      * :attr:`db_files` is a list of existing db files for the broker. This"},{"line_number":315,"context_line":"        list should have at least one entry for an initialised broker, and will"},{"line_number":316,"context_line":"        have two entries while a broker is in SHARDING state."},{"line_number":317,"context_line":"      * :attr:`db_file` is the path to whichever db is currently authoritative"},{"line_number":318,"context_line":"        for the container. Depending on the container\u0027s state, this may not be"},{"line_number":319,"context_line":"        the same as the ``db_file`` argument given to :meth:`~__init__`, unless"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_98e4de0e","line":316,"range":{"start_line":316,"start_character":13,"end_line":316,"end_character":24},"in_reply_to":"5f7c97a3_6ad8167a","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":320,"context_line":"        ``force_db_file`` is True in which case :attr:`db_file` is always equal"},{"line_number":321,"context_line":"        to the ``db_file`` argument given to :meth:`~__init__`."},{"line_number":322,"context_line":"      * :attr:`pending_file` is always equal to :attr:`_db_file` extended with"},{"line_number":323,"context_line":"        `.pending`, i.e. `\u003chash\u003e.db.pending`."},{"line_number":324,"context_line":"    \"\"\""},{"line_number":325,"context_line":"    db_type \u003d \u0027container\u0027"},{"line_number":326,"context_line":"    db_contains_type \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_4a3afa98","line":323,"range":{"start_line":323,"start_character":8,"end_line":323,"end_character":44},"updated":"2018-05-14 23:32:48.000000000","message":"nit: worth double-ticking these, to make them fixed-width?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":320,"context_line":"        ``force_db_file`` is True in which case :attr:`db_file` is always equal"},{"line_number":321,"context_line":"        to the ``db_file`` argument given to :meth:`~__init__`."},{"line_number":322,"context_line":"      * :attr:`pending_file` is always equal to :attr:`_db_file` extended with"},{"line_number":323,"context_line":"        `.pending`, i.e. `\u003chash\u003e.db.pending`."},{"line_number":324,"context_line":"    \"\"\""},{"line_number":325,"context_line":"    db_type \u003d \u0027container\u0027"},{"line_number":326,"context_line":"    db_contains_type \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_b8b43af3","line":323,"range":{"start_line":323,"start_character":8,"end_line":323,"end_character":44},"in_reply_to":"5f7c97a3_4a3afa98","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":336,"context_line":"        else:"},{"line_number":337,"context_line":"            db_dir \u003d os.path.dirname(db_file)"},{"line_number":338,"context_line":"            hash_, other, ext \u003d parse_db_filename(db_file)"},{"line_number":339,"context_line":"            base_db_file \u003d os.path.join(db_dir, hash_ + ext)"},{"line_number":340,"context_line":"        super(ContainerBroker, self).__init__("},{"line_number":341,"context_line":"            base_db_file, timeout, logger, account, container, pending_timeout,"},{"line_number":342,"context_line":"            stale_reads_ok, skip_commits\u003dskip_commits)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_0a5962fc","line":339,"updated":"2018-05-14 23:32:48.000000000","message":"Seems like a place where\n\n make_db_file_path(db_path, None)\n\nmight kinda make sense...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":368,"context_line":"        db_dir \u003d storage_directory(DATADIR, part, hsh)"},{"line_number":369,"context_line":"        db_path \u003d os.path.join(device_path, db_dir, hsh + \u0027.db\u0027)"},{"line_number":370,"context_line":"        if epoch:"},{"line_number":371,"context_line":"            db_path \u003d make_db_file_path(db_path, epoch)"},{"line_number":372,"context_line":"        broker \u003d ContainerBroker(db_path, account\u003daccount, container\u003dcontainer,"},{"line_number":373,"context_line":"                                 logger\u003dlogger)"},{"line_number":374,"context_line":"        if not broker._db_exists():"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_0a272266","line":371,"updated":"2018-05-14 23:32:48.000000000","message":"...and this could become\n\n db_path \u003d make_db_file_path(\n     os.path.join(device_path, db_dir, hsh + \u0027.db\u0027), epoch)","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":388,"context_line":"        if self.db_epoch is None:"},{"line_number":389,"context_line":"            # never been sharded"},{"line_number":390,"context_line":"            return UNSHARDED"},{"line_number":391,"context_line":"        if self.db_epoch !\u003d self._own_shard_range().epoch:"},{"line_number":392,"context_line":"            return UNSHARDED"},{"line_number":393,"context_line":"        if not self.get_shard_ranges():"},{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_4ac19ae4","line":393,"range":{"start_line":391,"start_character":28,"end_line":393,"end_character":38},"updated":"2018-05-14 23:32:48.000000000","message":"So it\u0027s these two DB hits that make us shy away from making this\n\n @property\n def db_state(self):\n\nyeah?\n\n... I think I like that.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":388,"context_line":"        if self.db_epoch is None:"},{"line_number":389,"context_line":"            # never been sharded"},{"line_number":390,"context_line":"            return UNSHARDED"},{"line_number":391,"context_line":"        if self.db_epoch !\u003d self._own_shard_range().epoch:"},{"line_number":392,"context_line":"            return UNSHARDED"},{"line_number":393,"context_line":"        if not self.get_shard_ranges():"},{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_d8b336e8","line":393,"range":{"start_line":391,"start_character":28,"end_line":393,"end_character":38},"in_reply_to":"5f7c97a3_4ac19ae4","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":391,"context_line":"        if self.db_epoch !\u003d self._own_shard_range().epoch:"},{"line_number":392,"context_line":"            return UNSHARDED"},{"line_number":393,"context_line":"        if not self.get_shard_ranges():"},{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def get_db_state_text(self, state\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_2a9bfe65","line":394,"range":{"start_line":394,"start_character":19,"end_line":394,"end_character":28},"updated":"2018-05-14 23:32:48.000000000","message":"I\u0027m not sure we really need to distinguish this from UNSHARDED...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":391,"context_line":"        if self.db_epoch !\u003d self._own_shard_range().epoch:"},{"line_number":392,"context_line":"            return UNSHARDED"},{"line_number":393,"context_line":"        if not self.get_shard_ranges():"},{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def get_db_state_text(self, state\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_98a05e92","line":394,"range":{"start_line":394,"start_character":19,"end_line":394,"end_character":28},"in_reply_to":"5f7c97a3_2a9bfe65","updated":"2018-05-15 17:46:09.000000000","message":"no harm in doing so, might help a human looking at db state","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def get_db_state_text(self, state\u003dNone):"},{"line_number":398,"context_line":"        if state is None:"},{"line_number":399,"context_line":"            state \u003d self.get_db_state()"},{"line_number":400,"context_line":"        return db_state_text(state)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_2a853ecf","line":397,"range":{"start_line":397,"start_character":32,"end_line":397,"end_character":42},"updated":"2018-05-14 23:32:48.000000000","message":"Feels strange -- if we\u0027re not getting the text for the broker\u0027s current state, why wouldn\u0027t the caller just call db_state_text itself?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":394,"context_line":"            return COLLAPSED"},{"line_number":395,"context_line":"        return SHARDED"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def get_db_state_text(self, state\u003dNone):"},{"line_number":398,"context_line":"        if state is None:"},{"line_number":399,"context_line":"            state \u003d self.get_db_state()"},{"line_number":400,"context_line":"        return db_state_text(state)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_b80c3a94","line":397,"range":{"start_line":397,"start_character":32,"end_line":397,"end_character":42},"in_reply_to":"5f7c97a3_2a853ecf","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":428,"context_line":"        \"\"\""},{"line_number":429,"context_line":"        if not self._db_files:"},{"line_number":430,"context_line":"            self.reload_db_files()"},{"line_number":431,"context_line":"        return self._db_files"},{"line_number":432,"context_line":""},{"line_number":433,"context_line":"    @property"},{"line_number":434,"context_line":"    def db_file(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_aa286e96","line":431,"updated":"2018-05-14 23:32:48.000000000","message":"Somewhat interesting that this doesn\u0027t consider self._force_db_file at all -- I\u0027m not sure whether that\u0027s good or bad yet.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":481,"context_line":"        self.create_container_info_table(conn, put_timestamp,"},{"line_number":482,"context_line":"                                         storage_policy_index)"},{"line_number":483,"context_line":"        self.create_shard_ranges_table(conn)"},{"line_number":484,"context_line":"        self._db_files \u003d None"},{"line_number":485,"context_line":""},{"line_number":486,"context_line":"    def create_object_table(self, conn):"},{"line_number":487,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_252491ae","line":484,"updated":"2018-05-14 23:32:48.000000000","message":"Right, since we just created a new DB file, gotta invalidate the cache.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        # *any* in-progress cursor will otherwise trip a \"database is locked\""},{"line_number":575,"context_line":"        # error."},{"line_number":576,"context_line":"        conn.execute(\"\"\""},{"line_number":577,"context_line":"            CREATE TABLE shard_ranges ("},{"line_number":578,"context_line":"                ROWID INTEGER PRIMARY KEY AUTOINCREMENT,"},{"line_number":579,"context_line":"                name TEXT,"},{"line_number":580,"context_line":"                lower TEXT,"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_055955fc","line":577,"range":{"start_line":577,"start_character":25,"end_line":577,"end_character":37},"updated":"2018-05-14 23:32:48.000000000","message":"Maybe better as \"shard_range\"? After all, it\u0027s the \"object\" table, not \"objects\", right?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        # *any* in-progress cursor will otherwise trip a \"database is locked\""},{"line_number":575,"context_line":"        # error."},{"line_number":576,"context_line":"        conn.execute(\"\"\""},{"line_number":577,"context_line":"            CREATE TABLE shard_ranges ("},{"line_number":578,"context_line":"                ROWID INTEGER PRIMARY KEY AUTOINCREMENT,"},{"line_number":579,"context_line":"                name TEXT,"},{"line_number":580,"context_line":"                lower TEXT,"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_584966b4","line":577,"range":{"start_line":577,"start_character":25,"end_line":577,"end_character":37},"in_reply_to":"5f7c97a3_055955fc","updated":"2018-05-15 17:46:09.000000000","message":"Done, but TBH the singular form seems weird to me.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        # *any* in-progress cursor will otherwise trip a \"database is locked\""},{"line_number":575,"context_line":"        # error."},{"line_number":576,"context_line":"        conn.execute(\"\"\""},{"line_number":577,"context_line":"            CREATE TABLE shard_ranges ("},{"line_number":578,"context_line":"                ROWID INTEGER PRIMARY KEY AUTOINCREMENT,"},{"line_number":579,"context_line":"                name TEXT,"},{"line_number":580,"context_line":"                lower TEXT,"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_50e5b1da","line":577,"range":{"start_line":577,"start_character":25,"end_line":577,"end_character":37},"in_reply_to":"5f7c97a3_584966b4","updated":"2018-05-15 23:27:29.000000000","message":"I agree, but I don\u0027t think we\u0027ll ever be able to fix \"object\", so... thanks.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":586,"context_line":"                state INTEGER,"},{"line_number":587,"context_line":"                state_timestamp TEXT,"},{"line_number":588,"context_line":"                epoch TEXT,"},{"line_number":589,"context_line":"                deleted INTEGER DEFAULT 0"},{"line_number":590,"context_line":"            );"},{"line_number":591,"context_line":"        \"\"\")"},{"line_number":592,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_c5b11d49","line":589,"updated":"2018-05-14 23:32:48.000000000","message":"Yet another ordering of these fields...\n\nBefore merging, it might be nice to make sure\n\n- ShardRange.__init__\u0027s args,\n- SHARD_RANGE_KEYS, and\n- create_shard_ranges_table\n\nto all agree on the same order, preferably (in my mind) related to the lifecycles of the different fields. Maybe\n\n\u003e (ROWID,) name, timestamp, lower, upper, deleted, meta_timestamp, object_count, bytes_used, state_timestamp, state, epoch\n\n?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":586,"context_line":"                state INTEGER,"},{"line_number":587,"context_line":"                state_timestamp TEXT,"},{"line_number":588,"context_line":"                epoch TEXT,"},{"line_number":589,"context_line":"                deleted INTEGER DEFAULT 0"},{"line_number":590,"context_line":"            );"},{"line_number":591,"context_line":"        \"\"\")"},{"line_number":592,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_53edc9b4","line":589,"in_reply_to":"5f7c97a3_c5b11d49","updated":"2018-05-15 17:46:09.000000000","message":"I\u0027ve made create_shard_range_table consistent but I\u0027m not inclined to re-order the ShardRange __init__ at this point.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":588,"context_line":"                epoch TEXT,"},{"line_number":589,"context_line":"                deleted INTEGER DEFAULT 0"},{"line_number":590,"context_line":"            );"},{"line_number":591,"context_line":"        \"\"\")"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"    def get_db_version(self, conn):"},{"line_number":594,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_453aed98","line":591,"updated":"2018-05-14 23:32:48.000000000","message":"I feel like we *at least* want an index on deleted for this table... probably dual-keyed as one of\n\n- (deleted, name)\n- (deleted, upper)\n- (deleted, lower)\n\nbut I\u0027m not sure which :-/\n\nMaybe we could add that after the fact? Looks like we didn\u0027t originally have the (deleted, name) index on the object table...\n\nShould there be a\n\n            CREATE TRIGGER shard_range_update BEFORE UPDATE ON shard_ranges\n            BEGIN\n                SELECT RAISE(FAIL, \u0027UPDATE not allowed; DELETE and INSERT\u0027);\n            END;\n\n?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":588,"context_line":"                epoch TEXT,"},{"line_number":589,"context_line":"                deleted INTEGER DEFAULT 0"},{"line_number":590,"context_line":"            );"},{"line_number":591,"context_line":"        \"\"\")"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"    def get_db_version(self, conn):"},{"line_number":594,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_f31655a8","line":591,"in_reply_to":"5f7c97a3_453aed98","updated":"2018-05-15 17:46:09.000000000","message":"been on the backlog [1] but I\u0027m also not sure what the index should be\n\n[1] https://trello.com/c/ZoNLw92h","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":747,"context_line":"            SELECT put_timestamp, delete_timestamp, object_count"},{"line_number":748,"context_line":"            FROM container_stat\u0027\u0027\u0027).fetchone()"},{"line_number":749,"context_line":"        info \u003d dict(info)"},{"line_number":750,"context_line":"        info.update(self._get_alternate_object_stats()[1])"},{"line_number":751,"context_line":"        return self._is_deleted_info(**info)"},{"line_number":752,"context_line":""},{"line_number":753,"context_line":"    def is_reclaimable(self, now, reclaim_age):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_c076eb70","line":750,"updated":"2018-05-14 23:32:48.000000000","message":"Hmm... so sometimes info will have bytes_used, and sometimes it won\u0027t... but that\u0027s not so crazy because _is_deleted_info intentionally takes an arbitrary info dict.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":831,"context_line":"    def _get_alternate_object_stats(self):"},{"line_number":832,"context_line":"        state \u003d self.get_db_state()"},{"line_number":833,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":834,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":835,"context_line":"            stats \u003d {\u0027object_count\u0027: other_info.get(\u0027object_count\u0027, 0),"},{"line_number":836,"context_line":"                     \u0027bytes_used\u0027: other_info.get(\u0027bytes_used\u0027, 0)}"},{"line_number":837,"context_line":"        elif state \u003d\u003d SHARDED and self.is_root_container():"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_80e313a4","line":834,"range":{"start_line":834,"start_character":47,"end_line":834,"end_character":56},"updated":"2018-05-14 23:32:48.000000000","message":"This broker should be read-only, yeah? Should we go straight to _do_get_info_query instead? The only real difference is in the extra call to _commit_puts_stale_ok... which makes me a little bit nervous...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":831,"context_line":"    def _get_alternate_object_stats(self):"},{"line_number":832,"context_line":"        state \u003d self.get_db_state()"},{"line_number":833,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":834,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":835,"context_line":"            stats \u003d {\u0027object_count\u0027: other_info.get(\u0027object_count\u0027, 0),"},{"line_number":836,"context_line":"                     \u0027bytes_used\u0027: other_info.get(\u0027bytes_used\u0027, 0)}"},{"line_number":837,"context_line":"        elif state \u003d\u003d SHARDED and self.is_root_container():"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_b3cf7d27","line":834,"range":{"start_line":834,"start_character":47,"end_line":834,"end_character":56},"in_reply_to":"5f7c97a3_80e313a4","updated":"2018-05-15 17:46:09.000000000","message":"the broker should have skip_commits \u003d\u003d True so calling _get_info() seems ok, no?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":833,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":834,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":835,"context_line":"            stats \u003d {\u0027object_count\u0027: other_info.get(\u0027object_count\u0027, 0),"},{"line_number":836,"context_line":"                     \u0027bytes_used\u0027: other_info.get(\u0027bytes_used\u0027, 0)}"},{"line_number":837,"context_line":"        elif state \u003d\u003d SHARDED and self.is_root_container():"},{"line_number":838,"context_line":"            stats \u003d self.get_shard_usage()"},{"line_number":839,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_8091d3f3","line":836,"range":{"start_line":836,"start_character":46,"end_line":836,"end_character":49},"updated":"2018-05-14 23:32:48.000000000","message":"Shouldn\u0027t these always be present? Even if they weren\u0027t, is 0 the best default? That seems to risk allowing a SHARDING db to report is_deleted as True, even if there are misplaced objects in the epoch DB...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":833,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":834,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":835,"context_line":"            stats \u003d {\u0027object_count\u0027: other_info.get(\u0027object_count\u0027, 0),"},{"line_number":836,"context_line":"                     \u0027bytes_used\u0027: other_info.get(\u0027bytes_used\u0027, 0)}"},{"line_number":837,"context_line":"        elif state \u003d\u003d SHARDED and self.is_root_container():"},{"line_number":838,"context_line":"            stats \u003d self.get_shard_usage()"},{"line_number":839,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_f39735eb","line":836,"range":{"start_line":836,"start_character":46,"end_line":836,"end_character":49},"in_reply_to":"5f7c97a3_8091d3f3","updated":"2018-05-15 17:46:09.000000000","message":"IIRC a SHARDED broker reports is_deleted True even when it has misplaced objects","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":999,"context_line":"                     the path"},{"line_number":1000,"context_line":"        :param storage_policy_index: storage policy index for query"},{"line_number":1001,"context_line":"        :param reverse: reverse the result order."},{"line_number":1002,"context_line":"        :param include_deleted: if True, include only deleted objects; if"},{"line_number":1003,"context_line":"            False (default), include only undeleted objects; otherwise, include"},{"line_number":1004,"context_line":"            both deleted and undeleted objects."},{"line_number":1005,"context_line":"        :param since_row: include only items whose ROWID is greater than"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_8058f355","line":1002,"range":{"start_line":1002,"start_character":32,"end_line":1002,"end_character":69},"updated":"2018-05-14 23:32:48.000000000","message":"That\u0027s very different from how include_deleted works for get_shard_ranges...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1007,"context_line":"        :param transform_func: an optional function that if given will be"},{"line_number":1008,"context_line":"            called for each object to get a transformed version of the object"},{"line_number":1009,"context_line":"            to include in the listing; should have same signature as"},{"line_number":1010,"context_line":"            :meth:`~_transform_record`; defaults to :meth:`~_transform_record`."},{"line_number":1011,"context_line":"        :param all_policies: if True, include objects for all storage policies"},{"line_number":1012,"context_line":"            ignoring any value given for ``storage_policy_index``"},{"line_number":1013,"context_line":"        :returns: list of tuples of (name, created_at, size, content_type,"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_dbcd7e85","line":1010,"updated":"2018-05-14 23:32:48.000000000","message":"It kinda feels like ContainerController.GET should be passing in transform_func\u003dself.update_data_record -- but we can\u0027t because we need _transform_record *first* :-/","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1042,"context_line":"        with self.get() as conn:"},{"line_number":1043,"context_line":"            results \u003d []"},{"line_number":1044,"context_line":"            if self.get_db_version(conn) \u003c 1:"},{"line_number":1045,"context_line":"                deleted_key \u003d \u0027+deleted\u0027"},{"line_number":1046,"context_line":"            else:"},{"line_number":1047,"context_line":"                deleted_key \u003d \u0027deleted\u0027"},{"line_number":1048,"context_line":"            while len(results) \u003c limit:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_602037ad","line":1045,"updated":"2018-05-14 23:32:48.000000000","message":"At some point I really ought to invest the time in understanding the history of why this is necessary...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1047,"context_line":"                deleted_key \u003d \u0027deleted\u0027"},{"line_number":1048,"context_line":"            while len(results) \u003c limit:"},{"line_number":1049,"context_line":"                query_keys \u003d [\u0027name\u0027, \u0027created_at\u0027, \u0027size\u0027, \u0027content_type\u0027,"},{"line_number":1050,"context_line":"                              \u0027etag\u0027, deleted_key]"},{"line_number":1051,"context_line":"                query_args \u003d []"},{"line_number":1052,"context_line":"                query_conditions \u003d []"},{"line_number":1053,"context_line":"                if end_marker and (not prefix or end_marker \u003c end_prefix):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_c0102b5a","line":1050,"updated":"2018-05-14 23:32:48.000000000","message":"nit: These don\u0027t change during the loop, yeah? Should be able to pull this up and out of the while.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1047,"context_line":"                deleted_key \u003d \u0027deleted\u0027"},{"line_number":1048,"context_line":"            while len(results) \u003c limit:"},{"line_number":1049,"context_line":"                query_keys \u003d [\u0027name\u0027, \u0027created_at\u0027, \u0027size\u0027, \u0027content_type\u0027,"},{"line_number":1050,"context_line":"                              \u0027etag\u0027, deleted_key]"},{"line_number":1051,"context_line":"                query_args \u003d []"},{"line_number":1052,"context_line":"                query_conditions \u003d []"},{"line_number":1053,"context_line":"                if end_marker and (not prefix or end_marker \u003c end_prefix):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_932281b8","line":1050,"in_reply_to":"5f7c97a3_c0102b5a","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1107,"context_line":"                # is no delimiter then we can simply return the result as"},{"line_number":1108,"context_line":"                # prefixes are now handled in the SQL statement."},{"line_number":1109,"context_line":"                if prefix is None or not delimiter:"},{"line_number":1110,"context_line":"                    return [transform_func(r, storage_policy_index)"},{"line_number":1111,"context_line":"                            for r in curs]"},{"line_number":1112,"context_line":""},{"line_number":1113,"context_line":"                # We have a delimiter and a prefix (possibly empty string) to"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_db39fed6","line":1110,"range":{"start_line":1110,"start_character":46,"end_line":1110,"end_character":66},"updated":"2018-05-14 23:32:48.000000000","message":"OK, so this is the policy that was requested, not necessarily the policy of the row...\n\nI wonder if it would be better to have\n\n query, args \u003d build_query(\n     query_keys + [\u00270 as storage_policy_index\u0027], query_conditions, query_args)\n\nwhen handling legacy DBs...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1107,"context_line":"                # is no delimiter then we can simply return the result as"},{"line_number":1108,"context_line":"                # prefixes are now handled in the SQL statement."},{"line_number":1109,"context_line":"                if prefix is None or not delimiter:"},{"line_number":1110,"context_line":"                    return [transform_func(r, storage_policy_index)"},{"line_number":1111,"context_line":"                            for r in curs]"},{"line_number":1112,"context_line":""},{"line_number":1113,"context_line":"                # We have a delimiter and a prefix (possibly empty string) to"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_6eb49880","line":1110,"range":{"start_line":1110,"start_character":46,"end_line":1110,"end_character":66},"in_reply_to":"5f7c97a3_db39fed6","updated":"2018-05-15 17:46:09.000000000","message":"this change w.r.t. master has become unnecessary since get_objects now requests all policies","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1166,"context_line":"        :param end_marker: if set, objects with names greater than or equal to"},{"line_number":1167,"context_line":"            this value will not be included in the list."},{"line_number":1168,"context_line":"        :param include_deleted: if True, include only deleted objects; if"},{"line_number":1169,"context_line":"            False, include only undeleted objects; otherwise (default), include"},{"line_number":1170,"context_line":"            both deleted and undeleted objects."},{"line_number":1171,"context_line":"        :param since_row: include only items whose ROWID is greater than"},{"line_number":1172,"context_line":"            the given row id; by default all rows are included."},{"line_number":1173,"context_line":"        :return: a list of dicts, each describing an object."}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_fbb07adf","line":1170,"range":{"start_line":1169,"start_character":51,"end_line":1170,"end_character":46},"updated":"2018-05-14 23:32:48.000000000","message":"...and we\u0027ve determined that the performance in this case (currently) *sucks*, yeah?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1173,"context_line":"        :return: a list of dicts, each describing an object."},{"line_number":1174,"context_line":"        \"\"\""},{"line_number":1175,"context_line":"        def transform_record(record, policy_index):"},{"line_number":1176,"context_line":"            result \u003d self._record_to_dict(record)"},{"line_number":1177,"context_line":"            result.setdefault(\u0027storage_policy_index\u0027, policy_index)"},{"line_number":1178,"context_line":"            return result"},{"line_number":1179,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_7b01caae","line":1176,"range":{"start_line":1176,"start_character":26,"end_line":1176,"end_character":41},"updated":"2018-05-14 23:32:48.000000000","message":"So this thing that was previously only being used when merging listings is now *also* tied into how we use list_objects_iter...\n\nThis is really highlighting for me that the \"name, created_at, size, content_type, etag, deleted, storage_policy_index\" column order in list_objects_iter, _record_to_dict, and _really_merge_items ought to be defined at the module level...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1194,"context_line":"        content-type and meta timestamps and the metadata timestamp is used as"},{"line_number":1195,"context_line":"        the last-modified time value."},{"line_number":1196,"context_line":"        \"\"\""},{"line_number":1197,"context_line":"        t_data, t_ctype, t_meta \u003d decode_timestamps(record[1])"},{"line_number":1198,"context_line":"        return (record[0], t_meta.internal) + record[2:5]"},{"line_number":1199,"context_line":""},{"line_number":1200,"context_line":"    def _record_to_dict(self, rec):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_7b1a2a17","line":1197,"range":{"start_line":1197,"start_character":34,"end_line":1197,"end_character":51},"updated":"2018-05-14 23:32:48.000000000","message":"So we definitely *don\u0027t* want to split these out for get_objects, yeah?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1194,"context_line":"        content-type and meta timestamps and the metadata timestamp is used as"},{"line_number":1195,"context_line":"        the last-modified time value."},{"line_number":1196,"context_line":"        \"\"\""},{"line_number":1197,"context_line":"        t_data, t_ctype, t_meta \u003d decode_timestamps(record[1])"},{"line_number":1198,"context_line":"        return (record[0], t_meta.internal) + record[2:5]"},{"line_number":1199,"context_line":""},{"line_number":1200,"context_line":"    def _record_to_dict(self, rec):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_29345add","line":1197,"range":{"start_line":1197,"start_character":34,"end_line":1197,"end_character":51},"in_reply_to":"5f7c97a3_7b1a2a17","updated":"2018-05-15 17:46:09.000000000","message":"correct","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1195,"context_line":"        the last-modified time value."},{"line_number":1196,"context_line":"        \"\"\""},{"line_number":1197,"context_line":"        t_data, t_ctype, t_meta \u003d decode_timestamps(record[1])"},{"line_number":1198,"context_line":"        return (record[0], t_meta.internal) + record[2:5]"},{"line_number":1199,"context_line":""},{"line_number":1200,"context_line":"    def _record_to_dict(self, rec):"},{"line_number":1201,"context_line":"        if rec:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1bce96b3","line":1198,"updated":"2018-05-14 23:32:48.000000000","message":"I\u0027m getting a weird smell from ContainerBroker._transform_record, ContainerBroker._record_to_dict, and ContainerController.update_data_record -- who the hell\u0027s responsible for getting from a sqlite Row to something a client can consume!?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1292,"context_line":""},{"line_number":1293,"context_line":"    def merge_shard_ranges(self, shard_ranges):"},{"line_number":1294,"context_line":"        \"\"\""},{"line_number":1295,"context_line":"        Merge items into the object or shard ranges tables."},{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1be9b6b5","line":1295,"range":{"start_line":1295,"start_character":25,"end_line":1295,"end_character":58},"updated":"2018-05-14 23:32:48.000000000","message":"Just shard range table now.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1292,"context_line":""},{"line_number":1293,"context_line":"    def merge_shard_ranges(self, shard_ranges):"},{"line_number":1294,"context_line":"        \"\"\""},{"line_number":1295,"context_line":"        Merge items into the object or shard ranges tables."},{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_29621ad9","line":1295,"range":{"start_line":1295,"start_character":25,"end_line":1295,"end_character":58},"in_reply_to":"5f7c97a3_1be9b6b5","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1294,"context_line":"        \"\"\""},{"line_number":1295,"context_line":"        Merge items into the object or shard ranges tables."},{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"},{"line_number":1299,"context_line":"            a dict representation of a shard range having `SHARD_RANGE_KEYS`."},{"line_number":1300,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_fb339a60","line":1297,"range":{"start_line":1297,"start_character":29,"end_line":1297,"end_character":51},"updated":"2018-05-14 23:32:48.000000000","message":"... or an individual shard range","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1294,"context_line":"        \"\"\""},{"line_number":1295,"context_line":"        Merge items into the object or shard ranges tables."},{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"},{"line_number":1299,"context_line":"            a dict representation of a shard range having `SHARD_RANGE_KEYS`."},{"line_number":1300,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_495d1613","line":1297,"range":{"start_line":1297,"start_character":29,"end_line":1297,"end_character":51},"in_reply_to":"5f7c97a3_fb339a60","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"},{"line_number":1299,"context_line":"            a dict representation of a shard range having `SHARD_RANGE_KEYS`."},{"line_number":1300,"context_line":"        \"\"\""},{"line_number":1301,"context_line":"        if not shard_ranges:"},{"line_number":1302,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_dbcede58","line":1299,"range":{"start_line":1299,"start_character":59,"end_line":1299,"end_character":75},"updated":"2018-05-14 23:32:48.000000000","message":"nit: double backticks for fixed-width","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1296,"context_line":""},{"line_number":1297,"context_line":"        :param shard_ranges: a list of shard ranges; each entry in the list"},{"line_number":1298,"context_line":"            should be an instance of :class:`~swift.common.utils.ShardRange` or"},{"line_number":1299,"context_line":"            a dict representation of a shard range having `SHARD_RANGE_KEYS`."},{"line_number":1300,"context_line":"        \"\"\""},{"line_number":1301,"context_line":"        if not shard_ranges:"},{"line_number":1302,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_e99d02bc","line":1299,"range":{"start_line":1299,"start_character":59,"end_line":1299,"end_character":75},"in_reply_to":"5f7c97a3_dbcede58","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1323,"context_line":"                chunk \u003d [record[\u0027name\u0027] for record"},{"line_number":1324,"context_line":"                         in item_list[offset:offset + SQLITE_ARG_LIMIT]]"},{"line_number":1325,"context_line":"                records.update("},{"line_number":1326,"context_line":"                    (rec[0], rec) for rec in curs.execute("},{"line_number":1327,"context_line":"                        \u0027SELECT %s FROM shard_ranges \u0027"},{"line_number":1328,"context_line":"                        \u0027WHERE deleted IN (0, 1) AND name IN (%s)\u0027 %"},{"line_number":1329,"context_line":"                        (\u0027, \u0027.join(SHARD_RANGE_KEYS),"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_5bcdee2e","line":1326,"range":{"start_line":1326,"start_character":21,"end_line":1326,"end_character":27},"updated":"2018-05-14 23:32:48.000000000","message":"i.e., name","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1325,"context_line":"                records.update("},{"line_number":1326,"context_line":"                    (rec[0], rec) for rec in curs.execute("},{"line_number":1327,"context_line":"                        \u0027SELECT %s FROM shard_ranges \u0027"},{"line_number":1328,"context_line":"                        \u0027WHERE deleted IN (0, 1) AND name IN (%s)\u0027 %"},{"line_number":1329,"context_line":"                        (\u0027, \u0027.join(SHARD_RANGE_KEYS),"},{"line_number":1330,"context_line":"                         \u0027,\u0027.join(\u0027?\u0027 * len(chunk))), chunk))"},{"line_number":1331,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1b5e9675","line":1328,"range":{"start_line":1328,"start_character":31,"end_line":1328,"end_character":65},"updated":"2018-05-14 23:32:48.000000000","message":"An argument in favor of a (deleted, name) index...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1350,"context_line":"                curs.executemany("},{"line_number":1351,"context_line":"                    \u0027DELETE FROM shard_ranges WHERE deleted in (0, 1) \u0027"},{"line_number":1352,"context_line":"                    \u0027AND name \u003d ?\u0027,"},{"line_number":1353,"context_line":"                    ((item[\u0027name\u0027],) for item in to_delete.values()))"},{"line_number":1354,"context_line":"            if to_add:"},{"line_number":1355,"context_line":"                vals \u003d \u0027,\u0027.join(\u0027?\u0027 * len(SHARD_RANGE_KEYS))"},{"line_number":1356,"context_line":"                curs.executemany("}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_7b91caf0","line":1353,"range":{"start_line":1353,"start_character":21,"end_line":1353,"end_character":68},"updated":"2018-05-14 23:32:48.000000000","message":"nit: could just use\n\n (item_ident for item_ident in to_delete)","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1350,"context_line":"                curs.executemany("},{"line_number":1351,"context_line":"                    \u0027DELETE FROM shard_ranges WHERE deleted in (0, 1) \u0027"},{"line_number":1352,"context_line":"                    \u0027AND name \u003d ?\u0027,"},{"line_number":1353,"context_line":"                    ((item[\u0027name\u0027],) for item in to_delete.values()))"},{"line_number":1354,"context_line":"            if to_add:"},{"line_number":1355,"context_line":"                vals \u003d \u0027,\u0027.join(\u0027?\u0027 * len(SHARD_RANGE_KEYS))"},{"line_number":1356,"context_line":"                curs.executemany("}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_e93982a3","line":1353,"range":{"start_line":1353,"start_character":21,"end_line":1353,"end_character":68},"in_reply_to":"5f7c97a3_7b91caf0","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1520,"context_line":"                                              sync_timestamp)"},{"line_number":1521,"context_line":"        # populate instance cache, but use existing conn to avoid deadlock"},{"line_number":1522,"context_line":"        # when it has a pending update"},{"line_number":1523,"context_line":"        self._populate_instance_cache(conn\u003dconn)"},{"line_number":1524,"context_line":"        try:"},{"line_number":1525,"context_line":"            conn.execute(\u0027\u0027\u0027"},{"line_number":1526,"context_line":"                DELETE FROM shard_ranges WHERE deleted \u003d 1 AND timestamp \u003c ?"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_80f23399","line":1523,"updated":"2018-05-14 23:32:48.000000000","message":"Wait, but _populate_instance_cache goes straight to _do_get_info_query -- I thought only _get_info would try to commit pending updates?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1520,"context_line":"                                              sync_timestamp)"},{"line_number":1521,"context_line":"        # populate instance cache, but use existing conn to avoid deadlock"},{"line_number":1522,"context_line":"        # when it has a pending update"},{"line_number":1523,"context_line":"        self._populate_instance_cache(conn\u003dconn)"},{"line_number":1524,"context_line":"        try:"},{"line_number":1525,"context_line":"            conn.execute(\u0027\u0027\u0027"},{"line_number":1526,"context_line":"                DELETE FROM shard_ranges WHERE deleted \u003d 1 AND timestamp \u003c ?"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_891fae06","line":1523,"in_reply_to":"5f7c97a3_80f23399","updated":"2018-05-15 17:46:09.000000000","message":"I think the comment might be referring to the conn not the broker","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1520,"context_line":"                                              sync_timestamp)"},{"line_number":1521,"context_line":"        # populate instance cache, but use existing conn to avoid deadlock"},{"line_number":1522,"context_line":"        # when it has a pending update"},{"line_number":1523,"context_line":"        self._populate_instance_cache(conn\u003dconn)"},{"line_number":1524,"context_line":"        try:"},{"line_number":1525,"context_line":"            conn.execute(\u0027\u0027\u0027"},{"line_number":1526,"context_line":"                DELETE FROM shard_ranges WHERE deleted \u003d 1 AND timestamp \u003c ?"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_4edaee77","line":1523,"in_reply_to":"5f7c97a3_891fae06","updated":"2018-05-15 23:27:29.000000000","message":"Never mind, I was being dumb.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1525,"context_line":"            conn.execute(\u0027\u0027\u0027"},{"line_number":1526,"context_line":"                DELETE FROM shard_ranges WHERE deleted \u003d 1 AND timestamp \u003c ?"},{"line_number":1527,"context_line":"                AND name !\u003d ?"},{"line_number":1528,"context_line":"            \u0027\u0027\u0027, (sync_timestamp, self.path))"},{"line_number":1529,"context_line":"        except sqlite3.OperationalError as err:"},{"line_number":1530,"context_line":"            if \u0027no such table: shard_ranges\u0027 not in str(err):"},{"line_number":1531,"context_line":"                raise"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_3b4fd293","line":1528,"range":{"start_line":1528,"start_character":18,"end_line":1528,"end_character":32},"updated":"2018-05-14 23:32:48.000000000","message":"Why sync_timestamp and not age_timestamp? Given the docstring in db.py, neither is an obvious choice...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1525,"context_line":"            conn.execute(\u0027\u0027\u0027"},{"line_number":1526,"context_line":"                DELETE FROM shard_ranges WHERE deleted \u003d 1 AND timestamp \u003c ?"},{"line_number":1527,"context_line":"                AND name !\u003d ?"},{"line_number":1528,"context_line":"            \u0027\u0027\u0027, (sync_timestamp, self.path))"},{"line_number":1529,"context_line":"        except sqlite3.OperationalError as err:"},{"line_number":1530,"context_line":"            if \u0027no such table: shard_ranges\u0027 not in str(err):"},{"line_number":1531,"context_line":"                raise"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_4998b673","line":1528,"range":{"start_line":1528,"start_character":18,"end_line":1528,"end_character":32},"in_reply_to":"5f7c97a3_3b4fd293","updated":"2018-05-15 17:46:09.000000000","message":"sync_timestamp because of the two times offered by the replicator this one is the most cautious; this implementation chooses to expire shard ranges at whatever age is considered appropriate for sync points.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1536,"context_line":"        \"\"\""},{"line_number":1537,"context_line":"        Returns a list of shard range rows."},{"line_number":1538,"context_line":""},{"line_number":1539,"context_line":"        To get all shard ranges use ``include_own\u003dTrue``. To get only the"},{"line_number":1540,"context_line":"        broker\u0027s own shard range use ``include_own\u003dTrue`` and"},{"line_number":1541,"context_line":"        ``exclude_others\u003dTrue``."},{"line_number":1542,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_9b070519","line":1539,"range":{"start_line":1539,"start_character":15,"end_line":1539,"end_character":31},"updated":"2018-05-14 23:32:48.000000000","message":"All *live* shard ranges","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1563,"context_line":"        def prep_states(states):"},{"line_number":1564,"context_line":"            state_set \u003d set()"},{"line_number":1565,"context_line":"            if isinstance(states, (list, tuple, set)):"},{"line_number":1566,"context_line":"                state_set.update(set(states))"},{"line_number":1567,"context_line":"            elif states is not None:"},{"line_number":1568,"context_line":"                state_set.add(states)"},{"line_number":1569,"context_line":"            return state_set"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_7b06aa22","line":1566,"range":{"start_line":1566,"start_character":33,"end_line":1566,"end_character":36},"updated":"2018-05-14 23:32:48.000000000","message":"nit: can just be\n\n state_set.update(states)","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1563,"context_line":"        def prep_states(states):"},{"line_number":1564,"context_line":"            state_set \u003d set()"},{"line_number":1565,"context_line":"            if isinstance(states, (list, tuple, set)):"},{"line_number":1566,"context_line":"                state_set.update(set(states))"},{"line_number":1567,"context_line":"            elif states is not None:"},{"line_number":1568,"context_line":"                state_set.add(states)"},{"line_number":1569,"context_line":"            return state_set"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_4970b608","line":1566,"range":{"start_line":1566,"start_character":33,"end_line":1566,"end_character":36},"in_reply_to":"5f7c97a3_7b06aa22","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"        excluded_states \u003d prep_states(exclude_states)"},{"line_number":1572,"context_line":"        included_states \u003d prep_states(states)"},{"line_number":1573,"context_line":"        included_states -\u003d excluded_states"},{"line_number":1574,"context_line":""},{"line_number":1575,"context_line":"        def do_query(conn):"},{"line_number":1576,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_fb895974","line":1573,"updated":"2018-05-14 23:32:48.000000000","message":"So passing\n\n states\u003dx, exclude_states\u003dx\n\n... we get back *all* states?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"        excluded_states \u003d prep_states(exclude_states)"},{"line_number":1572,"context_line":"        included_states \u003d prep_states(states)"},{"line_number":1573,"context_line":"        included_states -\u003d excluded_states"},{"line_number":1574,"context_line":""},{"line_number":1575,"context_line":"        def do_query(conn):"},{"line_number":1576,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_eebdfa01","line":1573,"in_reply_to":"5f7c97a3_09687e1a","updated":"2018-05-15 23:27:29.000000000","message":"Still feels a little weird... I\u0027d kind expect a\n\n if included_states.issubset(excluded_states):\n     return []\n\nor even\n\n     raise ValueError\n\nbut w/e, shouldn\u0027t come up in practice, right? *shrug*","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"        excluded_states \u003d prep_states(exclude_states)"},{"line_number":1572,"context_line":"        included_states \u003d prep_states(states)"},{"line_number":1573,"context_line":"        included_states -\u003d excluded_states"},{"line_number":1574,"context_line":""},{"line_number":1575,"context_line":"        def do_query(conn):"},{"line_number":1576,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_09687e1a","line":1573,"in_reply_to":"5f7c97a3_fb895974","updated":"2018-05-15 17:46:09.000000000","message":"all excluding x","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1602,"context_line":"                data \u003d conn.execute(sql, params)"},{"line_number":1603,"context_line":"                data.row_factory \u003d None"},{"line_number":1604,"context_line":"                return [row for row in data]"},{"line_number":1605,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1606,"context_line":"                if \u0027no such table: shard_ranges\u0027 not in str(err):"},{"line_number":1607,"context_line":"                    raise"},{"line_number":1608,"context_line":"                return []"},{"line_number":1609,"context_line":""},{"line_number":1610,"context_line":"        if connection:"},{"line_number":1611,"context_line":"            return do_query(connection)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_bb23e1a6","line":1608,"range":{"start_line":1605,"start_character":12,"end_line":1608,"end_character":25},"updated":"2018-05-14 23:32:48.000000000","message":"nit: I might save some indenting here and handle the missing table outside of do_query","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        :param states: a list of values each of which may be the name of a"},{"line_number":1628,"context_line":"            state, the number of a state, or an alias"},{"line_number":1629,"context_line":"        :return: a set of integer state numbers"},{"line_number":1630,"context_line":"        :raises ValueError: if any value in the given list is neither a valid"},{"line_number":1631,"context_line":"            state nor a valid alias"},{"line_number":1632,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_dbeb3d10","line":1629,"updated":"2018-05-14 23:32:48.000000000","message":"...or None","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        :param states: a list of values each of which may be the name of a"},{"line_number":1628,"context_line":"            state, the number of a state, or an alias"},{"line_number":1629,"context_line":"        :return: a set of integer state numbers"},{"line_number":1630,"context_line":"        :raises ValueError: if any value in the given list is neither a valid"},{"line_number":1631,"context_line":"            state nor a valid alias"},{"line_number":1632,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_09371e01","line":1629,"in_reply_to":"5f7c97a3_dbeb3d10","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1689,"context_line":"            return []"},{"line_number":1690,"context_line":""},{"line_number":1691,"context_line":"        shard_ranges \u003d ["},{"line_number":1692,"context_line":"            ShardRange(*row)"},{"line_number":1693,"context_line":"            for row in self._get_shard_range_rows("},{"line_number":1694,"context_line":"                include_deleted\u003dinclude_deleted, states\u003dstates,"},{"line_number":1695,"context_line":"                exclude_states\u003dexclude_states, include_own\u003dinclude_own,"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_7b954986","line":1692,"updated":"2018-05-14 23:32:48.000000000","message":"At some point I should learn how to use row_factory to do this sort of thing for us...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1694,"context_line":"                include_deleted\u003dinclude_deleted, states\u003dstates,"},{"line_number":1695,"context_line":"                exclude_states\u003dexclude_states, include_own\u003dinclude_own,"},{"line_number":1696,"context_line":"                exclude_others\u003dexclude_others)]"},{"line_number":1697,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1698,"context_line":"        if includes:"},{"line_number":1699,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1700,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_fb44fa75","line":1697,"updated":"2018-05-14 23:32:48.000000000","message":"Maybe an argument in favor of a (deleted, upper) index?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1697,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1698,"context_line":"        if includes:"},{"line_number":1699,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1700,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"},{"line_number":1701,"context_line":"        else:"},{"line_number":1702,"context_line":"            if reverse:"},{"line_number":1703,"context_line":"                shard_ranges.reverse()"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1b6c559a","line":1700,"updated":"2018-05-14 23:32:48.000000000","message":"Should we just return here? We shouldn\u0027t really need to think about fill_gaps when we\u0027re dealing with includes, yeah?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1697,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1698,"context_line":"        if includes:"},{"line_number":1699,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1700,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"},{"line_number":1701,"context_line":"        else:"},{"line_number":1702,"context_line":"            if reverse:"},{"line_number":1703,"context_line":"                shard_ranges.reverse()"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_71079c19","line":1700,"in_reply_to":"5f7c97a3_1b6c559a","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1715,"context_line":"            else:"},{"line_number":1716,"context_line":"                if shard_ranges:"},{"line_number":1717,"context_line":"                    last_upper \u003d shard_ranges[-1].upper"},{"line_number":1718,"context_line":"                else:"},{"line_number":1719,"context_line":"                    last_upper \u003d marker or ShardRange.MIN"},{"line_number":1720,"context_line":"                required_upper \u003d end_marker or ShardRange.MAX"},{"line_number":1721,"context_line":"                filler_index \u003d len(shard_ranges)"},{"line_number":1722,"context_line":"            if required_upper \u003e last_upper:"},{"line_number":1723,"context_line":"                filler_sr \u003d self.get_own_shard_range()"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_fb3399a9","line":1720,"range":{"start_line":1718,"start_character":0,"end_line":1720,"end_character":61},"updated":"2018-05-14 23:32:48.000000000","message":"Common to both halves of the if -- not sure whether it\u0027s worth trying to DRY out though...","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1731,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_own\u003dTrue,"},{"line_number":1732,"context_line":"                                             include_deleted\u003dTrue,"},{"line_number":1733,"context_line":"                                             exclude_others\u003dTrue)"},{"line_number":1734,"context_line":"        now \u003d Timestamp.now()"},{"line_number":1735,"context_line":"        if shard_ranges:"},{"line_number":1736,"context_line":"            own_shard_range \u003d shard_ranges[0]"},{"line_number":1737,"context_line":"        elif no_default:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_3b1591e5","line":1734,"updated":"2018-05-14 23:32:48.000000000","message":"nit: only used in the else below","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1731,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_own\u003dTrue,"},{"line_number":1732,"context_line":"                                             include_deleted\u003dTrue,"},{"line_number":1733,"context_line":"                                             exclude_others\u003dTrue)"},{"line_number":1734,"context_line":"        now \u003d Timestamp.now()"},{"line_number":1735,"context_line":"        if shard_ranges:"},{"line_number":1736,"context_line":"            own_shard_range \u003d shard_ranges[0]"},{"line_number":1737,"context_line":"        elif no_default:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_b1d45449","line":1734,"in_reply_to":"5f7c97a3_3b1591e5","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1761,"context_line":"        if own_shard_range:"},{"line_number":1762,"context_line":"            info \u003d self.get_info()"},{"line_number":1763,"context_line":"            own_shard_range.update_meta("},{"line_number":1764,"context_line":"                info.get(\u0027object_count\u0027, 0), info.get(\u0027bytes_used\u0027, 0))"},{"line_number":1765,"context_line":"        return own_shard_range"},{"line_number":1766,"context_line":""},{"line_number":1767,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_dbbeddd3","line":1764,"range":{"start_line":1764,"start_character":21,"end_line":1764,"end_character":24},"updated":"2018-05-14 23:32:48.000000000","message":"When would these *not* be present?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1761,"context_line":"        if own_shard_range:"},{"line_number":1762,"context_line":"            info \u003d self.get_info()"},{"line_number":1763,"context_line":"            own_shard_range.update_meta("},{"line_number":1764,"context_line":"                info.get(\u0027object_count\u0027, 0), info.get(\u0027bytes_used\u0027, 0))"},{"line_number":1765,"context_line":"        return own_shard_range"},{"line_number":1766,"context_line":""},{"line_number":1767,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_d13c7094","line":1764,"range":{"start_line":1764,"start_character":21,"end_line":1764,"end_character":24},"in_reply_to":"5f7c97a3_dbbeddd3","updated":"2018-05-15 17:46:09.000000000","message":"Done","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1774,"context_line":"        self.merge_shard_ranges(own_shard_range)"},{"line_number":1775,"context_line":"        return own_shard_range"},{"line_number":1776,"context_line":""},{"line_number":1777,"context_line":"    def get_shard_usage(self):"},{"line_number":1778,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dSHARD_STATS_STATES)"},{"line_number":1779,"context_line":"        return {\u0027bytes_used\u0027: sum(sr.bytes_used for sr in shard_ranges),"},{"line_number":1780,"context_line":"                \u0027object_count\u0027: sum(sr.object_count for sr in shard_ranges)}"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1bc8d571","line":1777,"updated":"2018-05-14 23:32:48.000000000","message":"Would we ever call this on a shard (instead of just the root container)?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1774,"context_line":"        self.merge_shard_ranges(own_shard_range)"},{"line_number":1775,"context_line":"        return own_shard_range"},{"line_number":1776,"context_line":""},{"line_number":1777,"context_line":"    def get_shard_usage(self):"},{"line_number":1778,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dSHARD_STATS_STATES)"},{"line_number":1779,"context_line":"        return {\u0027bytes_used\u0027: sum(sr.bytes_used for sr in shard_ranges),"},{"line_number":1780,"context_line":"                \u0027object_count\u0027: sum(sr.object_count for sr in shard_ranges)}"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_1153c83a","line":1777,"in_reply_to":"5f7c97a3_1bc8d571","updated":"2018-05-15 17:46:09.000000000","message":"not anymore :) empty() might have called this on a shard but shouldn\u0027t - sub-shard stats are not updated in shards so they should not be considered","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1782,"context_line":"    def get_all_shard_range_data(self):"},{"line_number":1783,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":1784,"context_line":"                                             include_own\u003dTrue)"},{"line_number":1785,"context_line":"        return [dict(sr, record_type\u003dRECORD_TYPE_SHARD)"},{"line_number":1786,"context_line":"                for sr in shard_ranges]"},{"line_number":1787,"context_line":""},{"line_number":1788,"context_line":"    def set_sharding_state(self, epoch\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_8f70240a","line":1785,"range":{"start_line":1785,"start_character":25,"end_line":1785,"end_character":54},"updated":"2018-05-14 23:32:48.000000000","message":"Do we still need this now that we aren\u0027t sending *everything* through merge_items?","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1782,"context_line":"    def get_all_shard_range_data(self):"},{"line_number":1783,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":1784,"context_line":"                                             include_own\u003dTrue)"},{"line_number":1785,"context_line":"        return [dict(sr, record_type\u003dRECORD_TYPE_SHARD)"},{"line_number":1786,"context_line":"                for sr in shard_ranges]"},{"line_number":1787,"context_line":""},{"line_number":1788,"context_line":"    def set_sharding_state(self, epoch\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_54104e4e","line":1785,"range":{"start_line":1785,"start_character":25,"end_line":1785,"end_character":54},"in_reply_to":"5f7c97a3_8f70240a","updated":"2018-05-15 17:46:09.000000000","message":"No, I had meant to remove it.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"83b584da489c834a6faee978a30d9090359c75cf","unresolved":false,"context_lines":[{"line_number":1785,"context_line":"        return [dict(sr, record_type\u003dRECORD_TYPE_SHARD)"},{"line_number":1786,"context_line":"                for sr in shard_ranges]"},{"line_number":1787,"context_line":""},{"line_number":1788,"context_line":"    def set_sharding_state(self, epoch\u003dNone):"},{"line_number":1789,"context_line":"        epoch \u003d epoch or self.get_own_shard_range().epoch"},{"line_number":1790,"context_line":"        if not epoch:"},{"line_number":1791,"context_line":"            self.logger.warning(\"Container \u0027%s\u0027 cannot be set to sharding \""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_40509b0b","line":1788,"range":{"start_line":1788,"start_character":33,"end_line":1788,"end_character":43},"updated":"2018-05-14 23:32:48.000000000","message":"Only used in tests, and even then only once.","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fc4e0639b236d6a808fcc999c2e32870efed7d5d","unresolved":false,"context_lines":[{"line_number":1785,"context_line":"        return [dict(sr, record_type\u003dRECORD_TYPE_SHARD)"},{"line_number":1786,"context_line":"                for sr in shard_ranges]"},{"line_number":1787,"context_line":""},{"line_number":1788,"context_line":"    def set_sharding_state(self, epoch\u003dNone):"},{"line_number":1789,"context_line":"        epoch \u003d epoch or self.get_own_shard_range().epoch"},{"line_number":1790,"context_line":"        if not epoch:"},{"line_number":1791,"context_line":"            self.logger.warning(\"Container \u0027%s\u0027 cannot be set to sharding \""}],"source_content_type":"text/x-python","patch_set":7,"id":"5f7c97a3_145df613","line":1788,"range":{"start_line":1788,"start_character":33,"end_line":1788,"end_character":43},"in_reply_to":"5f7c97a3_40509b0b","updated":"2018-05-15 17:46:09.000000000","message":"and even then it is no longer required!","commit_id":"55b19d4e4b817a475b8f857ff374b9391866ef8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":577,"context_line":"                state_timestamp TEXT,"},{"line_number":578,"context_line":"                epoch TEXT"},{"line_number":579,"context_line":"            );"},{"line_number":580,"context_line":"        \"\"\" % SHARD_RANGE_TABLE)"},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"    def get_db_version(self, conn):"},{"line_number":583,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_f0a5dd9f","line":580,"updated":"2018-05-15 23:27:29.000000000","message":"Any thoughts on the update-trigger idea? Seems like it\u0027ll probably be necessary if we ever get to using sync points for shard ranges...","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":577,"context_line":"                state_timestamp TEXT,"},{"line_number":578,"context_line":"                epoch TEXT"},{"line_number":579,"context_line":"            );"},{"line_number":580,"context_line":"        \"\"\" % SHARD_RANGE_TABLE)"},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"    def get_db_version(self, conn):"},{"line_number":583,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_3cdebe3d","line":580,"in_reply_to":"5f7c97a3_f0a5dd9f","updated":"2018-05-16 18:49:30.000000000","message":"seemed like a good defensive measure but needs a test, might make it in next patchset","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":662,"context_line":"            return False"},{"line_number":663,"context_line":"        if self.is_root_container():"},{"line_number":664,"context_line":"            # sharded shards don\u0027t get updates from their shards so their shard"},{"line_number":665,"context_line":"            # usage should not be relied upon"},{"line_number":666,"context_line":"            return self.get_shard_usage()[\u0027object_count\u0027] \u003c\u003d 0"},{"line_number":667,"context_line":"        return True"},{"line_number":668,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_30f7d58d","line":665,"updated":"2018-05-15 23:27:29.000000000","message":"+1","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1092,"context_line":"                    if \u0027no such column: storage_policy_index\u0027 not in str(err):"},{"line_number":1093,"context_line":"                        raise"},{"line_number":1094,"context_line":"                    query, args \u003d build_query("},{"line_number":1095,"context_line":"                        query_keys, query_conditions, query_args)"},{"line_number":1096,"context_line":"                    curs \u003d conn.execute(query, tuple(args))"},{"line_number":1097,"context_line":"                curs.row_factory \u003d None"},{"line_number":1098,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_9097c9ec","line":1095,"updated":"2018-05-15 23:27:29.000000000","message":"http://paste.openstack.org/show/721044/ seems like an improvement -- then transform_func (almost; see below) always gets a consistent set of columns.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1092,"context_line":"                    if \u0027no such column: storage_policy_index\u0027 not in str(err):"},{"line_number":1093,"context_line":"                        raise"},{"line_number":1094,"context_line":"                    query, args \u003d build_query("},{"line_number":1095,"context_line":"                        query_keys, query_conditions, query_args)"},{"line_number":1096,"context_line":"                    curs \u003d conn.execute(query, tuple(args))"},{"line_number":1097,"context_line":"                curs.row_factory \u003d None"},{"line_number":1098,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_fc2126b5","line":1095,"in_reply_to":"5f7c97a3_9097c9ec","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1136,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1137,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1138,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1139,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1140,"context_line":"                        curs.close()"},{"line_number":1141,"context_line":"                        break"},{"line_number":1142,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_5307fb18","line":1139,"updated":"2018-05-15 23:27:29.000000000","message":"Should this include deleted and/or storage_policy? I guess deleted is trivially 0; I guess storage_policy can be 0? IDK, it\u0027s kinda nonsense... Maybe None, to make it more obvious that it\u0027s nonsense?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1136,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1137,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1138,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1139,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1140,"context_line":"                        curs.close()"},{"line_number":1141,"context_line":"                        break"},{"line_number":1142,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_77045115","line":1139,"in_reply_to":"5f7c97a3_5307fb18","updated":"2018-05-16 18:49:30.000000000","message":"? out of scope","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1136,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1137,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1138,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1139,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1140,"context_line":"                        curs.close()"},{"line_number":1141,"context_line":"                        break"},{"line_number":1142,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_f6bda302","line":1139,"in_reply_to":"5f7c97a3_77045115","updated":"2018-05-17 22:46:22.000000000","message":"Well, kind of... we\u0027re taking a situation where we already had some skew and making it *worse*.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1136,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1137,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1138,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1139,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1140,"context_line":"                        curs.close()"},{"line_number":1141,"context_line":"                        break"},{"line_number":1142,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_c72300be","line":1139,"in_reply_to":"5f7c97a3_f6bda302","updated":"2018-05-18 17:56:24.000000000","message":"get_objects never exercises this path, correct? so what did we make worse. sorry if I am missing it.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1171,"context_line":""},{"line_number":1172,"context_line":"        limit \u003d CONTAINER_LISTING_LIMIT if limit is None else limit"},{"line_number":1173,"context_line":"        return self.list_objects_iter("},{"line_number":1174,"context_line":"            limit, marker, end_marker, prefix\u003dNone, delimiter\u003dNone, path\u003dNone,"},{"line_number":1175,"context_line":"            reverse\u003dFalse, include_deleted\u003dinclude_deleted,"},{"line_number":1176,"context_line":"            transform_func\u003dtransform_record, since_row\u003dsince_row,"},{"line_number":1177,"context_line":"            all_policies\u003dTrue"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_30afd52a","line":1174,"range":{"start_line":1174,"start_character":52,"end_line":1174,"end_character":66},"updated":"2018-05-15 23:27:29.000000000","message":"Definitely good! I don\u0027t want _record_to_dict thinking about \"ok, so if content type is none, I gotta make a subdir...\"","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1171,"context_line":""},{"line_number":1172,"context_line":"        limit \u003d CONTAINER_LISTING_LIMIT if limit is None else limit"},{"line_number":1173,"context_line":"        return self.list_objects_iter("},{"line_number":1174,"context_line":"            limit, marker, end_marker, prefix\u003dNone, delimiter\u003dNone, path\u003dNone,"},{"line_number":1175,"context_line":"            reverse\u003dFalse, include_deleted\u003dinclude_deleted,"},{"line_number":1176,"context_line":"            transform_func\u003dtransform_record, since_row\u003dsince_row,"},{"line_number":1177,"context_line":"            all_policies\u003dTrue"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_370a591f","line":1174,"range":{"start_line":1174,"start_character":52,"end_line":1174,"end_character":66},"in_reply_to":"5f7c97a3_30afd52a","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1336,"context_line":"                if merge_shards(item, existing):"},{"line_number":1337,"context_line":"                    # exists with older timestamp"},{"line_number":1338,"context_line":"                    if item_ident in records:"},{"line_number":1339,"context_line":"                        to_delete[item_ident] \u003d item"},{"line_number":1340,"context_line":"                    # duplicate entries in item_list"},{"line_number":1341,"context_line":"                    if (item_ident not in to_add or"},{"line_number":1342,"context_line":"                            merge_shards(item, to_add[item_ident])):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6e7c8a71","line":1339,"range":{"start_line":1339,"start_character":48,"end_line":1339,"end_character":52},"updated":"2018-05-15 23:27:29.000000000","message":"Don\u0027t really care about item now... http://paste.openstack.org/show/721047/","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee8e06013003ae0b597ddd68df02f98159d5b9ae","unresolved":false,"context_lines":[{"line_number":1691,"context_line":"                include_deleted\u003dinclude_deleted, states\u003dstates,"},{"line_number":1692,"context_line":"                exclude_states\u003dexclude_states, include_own\u003dinclude_own,"},{"line_number":1693,"context_line":"                exclude_others\u003dexclude_others)]"},{"line_number":1694,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1695,"context_line":"        if includes:"},{"line_number":1696,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1697,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_32a02742","line":1694,"updated":"2018-05-16 16:54:46.000000000","message":"Might be worth noting that if we *don\u0027t* sort by upper first, we break a key assumption for bisect, which is used by find_shard_ranges.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1691,"context_line":"                include_deleted\u003dinclude_deleted, states\u003dstates,"},{"line_number":1692,"context_line":"                exclude_states\u003dexclude_states, include_own\u003dinclude_own,"},{"line_number":1693,"context_line":"                exclude_others\u003dexclude_others)]"},{"line_number":1694,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1695,"context_line":"        if includes:"},{"line_number":1696,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1697,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_97db1620","line":1694,"in_reply_to":"5f7c97a3_32a02742","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1694,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1695,"context_line":"        if includes:"},{"line_number":1696,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1697,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"},{"line_number":1698,"context_line":"        else:"},{"line_number":1699,"context_line":"            if reverse:"},{"line_number":1700,"context_line":"                shard_ranges.reverse()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ee52dadd","line":1697,"updated":"2018-05-15 23:27:29.000000000","message":"http://paste.openstack.org/show/721050/","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1694,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1695,"context_line":"        if includes:"},{"line_number":1696,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1697,"context_line":"            shard_ranges \u003d [shard_range] if shard_range else []"},{"line_number":1698,"context_line":"        else:"},{"line_number":1699,"context_line":"            if reverse:"},{"line_number":1700,"context_line":"                shard_ranges.reverse()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_97e48d3d","line":1697,"in_reply_to":"5f7c97a3_ee52dadd","updated":"2018-05-16 18:49:30.000000000","message":"oops, I remember making that change, must\u0027ve got lost :/","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1821,"context_line":""},{"line_number":1822,"context_line":"        # firstly lets ensure we have a connection, otherwise we could start"},{"line_number":1823,"context_line":"        # playing with the wrong database while setting up the shard range"},{"line_number":1824,"context_line":"        # database"},{"line_number":1825,"context_line":"        if not self.conn:"},{"line_number":1826,"context_line":"            with self.get():"},{"line_number":1827,"context_line":"                pass"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_8e29e637","line":1824,"updated":"2018-05-15 23:27:29.000000000","message":"This doesn\u0027t quite make sense to me... isn\u0027t one of the first things we do in get_info() a call to get() a connection?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1821,"context_line":""},{"line_number":1822,"context_line":"        # firstly lets ensure we have a connection, otherwise we could start"},{"line_number":1823,"context_line":"        # playing with the wrong database while setting up the shard range"},{"line_number":1824,"context_line":"        # database"},{"line_number":1825,"context_line":"        if not self.conn:"},{"line_number":1826,"context_line":"            with self.get():"},{"line_number":1827,"context_line":"                pass"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_d7cd2566","line":1824,"in_reply_to":"5f7c97a3_8e29e637","updated":"2018-05-16 18:49:30.000000000","message":"+1 these comments are outdated","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1827,"context_line":"                pass"},{"line_number":1828,"context_line":""},{"line_number":1829,"context_line":"        # For this initial version, we\u0027ll create a new container along side."},{"line_number":1830,"context_line":"        # Later we will remove parts so the shard range DB only has what it"},{"line_number":1831,"context_line":"        # really needs"},{"line_number":1832,"context_line":"        info \u003d self.get_info()"},{"line_number":1833,"context_line":""},{"line_number":1834,"context_line":"        # The tmp_dir is cleaned up by the replicators after reclaim_age, so"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_4e1f4e07","line":1831,"range":{"start_line":1830,"start_character":10,"end_line":1831,"end_character":22},"updated":"2018-05-15 23:27:29.000000000","message":"Is this still up for consideration? I think the comment may be outdated...","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1827,"context_line":"                pass"},{"line_number":1828,"context_line":""},{"line_number":1829,"context_line":"        # For this initial version, we\u0027ll create a new container along side."},{"line_number":1830,"context_line":"        # Later we will remove parts so the shard range DB only has what it"},{"line_number":1831,"context_line":"        # really needs"},{"line_number":1832,"context_line":"        info \u003d self.get_info()"},{"line_number":1833,"context_line":""},{"line_number":1834,"context_line":"        # The tmp_dir is cleaned up by the replicators after reclaim_age, so"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_f7c86155","line":1831,"range":{"start_line":1830,"start_character":10,"end_line":1831,"end_character":22},"in_reply_to":"5f7c97a3_4e1f4e07","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1840,"context_line":"        tmp_db_file \u003d os.path.join(tmp_dir, \"preshard%s.db\" % str(uuid4()))"},{"line_number":1841,"context_line":"        sub_broker \u003d ContainerBroker(tmp_db_file, self.timeout,"},{"line_number":1842,"context_line":"                                     self.logger, self.account, self.container,"},{"line_number":1843,"context_line":"                                     self.pending_timeout, self.stale_reads_ok)"},{"line_number":1844,"context_line":"        sub_broker.initialize(info[\u0027put_timestamp\u0027],"},{"line_number":1845,"context_line":"                              info[\u0027storage_policy_index\u0027])"},{"line_number":1846,"context_line":"        sub_broker.update_metadata(self.metadata)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0e15d6e4","line":1843,"range":{"start_line":1843,"start_character":59,"end_line":1843,"end_character":78},"updated":"2018-05-15 23:27:29.000000000","message":"We\u0027re working with a brand new DB created especially for this purpose -- do we really want to allow stale reads on it? There should never even be a pending file for it, yeah?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1840,"context_line":"        tmp_db_file \u003d os.path.join(tmp_dir, \"preshard%s.db\" % str(uuid4()))"},{"line_number":1841,"context_line":"        sub_broker \u003d ContainerBroker(tmp_db_file, self.timeout,"},{"line_number":1842,"context_line":"                                     self.logger, self.account, self.container,"},{"line_number":1843,"context_line":"                                     self.pending_timeout, self.stale_reads_ok)"},{"line_number":1844,"context_line":"        sub_broker.initialize(info[\u0027put_timestamp\u0027],"},{"line_number":1845,"context_line":"                              info[\u0027storage_policy_index\u0027])"},{"line_number":1846,"context_line":"        sub_broker.update_metadata(self.metadata)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_f752a166","line":1843,"range":{"start_line":1843,"start_character":59,"end_line":1843,"end_character":78},"in_reply_to":"5f7c97a3_0e15d6e4","updated":"2018-05-16 18:49:30.000000000","message":"I think it\u0027s irrelevant - this broker isn\u0027t used for any reads. pending_timeout is also unnecessary.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1851,"context_line":"        sub_broker.merge_shard_ranges("},{"line_number":1852,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1853,"context_line":""},{"line_number":1854,"context_line":"        # We also need to sync the sync tables as we have no idea how long"},{"line_number":1855,"context_line":"        # sharding will take and we want to be able to continue replication"},{"line_number":1856,"context_line":"        # (albeit we only ever usync in sharding state, but we need to keep"},{"line_number":1857,"context_line":"        # the sync point mutable)"},{"line_number":1858,"context_line":"        for incoming in (True, False):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6edb2a4e","line":1855,"range":{"start_line":1854,"start_character":50,"end_line":1855,"end_character":28},"updated":"2018-05-15 23:27:29.000000000","message":"Yay! We have some idea now, and it doesn\u0027t seem *completely* terrible! Still a good idea :-)","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1851,"context_line":"        sub_broker.merge_shard_ranges("},{"line_number":1852,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1853,"context_line":""},{"line_number":1854,"context_line":"        # We also need to sync the sync tables as we have no idea how long"},{"line_number":1855,"context_line":"        # sharding will take and we want to be able to continue replication"},{"line_number":1856,"context_line":"        # (albeit we only ever usync in sharding state, but we need to keep"},{"line_number":1857,"context_line":"        # the sync point mutable)"},{"line_number":1858,"context_line":"        for incoming in (True, False):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_b7a4294c","line":1855,"range":{"start_line":1854,"start_character":50,"end_line":1855,"end_character":28},"in_reply_to":"5f7c97a3_6edb2a4e","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1852,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1853,"context_line":""},{"line_number":1854,"context_line":"        # We also need to sync the sync tables as we have no idea how long"},{"line_number":1855,"context_line":"        # sharding will take and we want to be able to continue replication"},{"line_number":1856,"context_line":"        # (albeit we only ever usync in sharding state, but we need to keep"},{"line_number":1857,"context_line":"        # the sync point mutable)"},{"line_number":1858,"context_line":"        for incoming in (True, False):"},{"line_number":1859,"context_line":"            syncs \u003d self.get_syncs(incoming)"},{"line_number":1860,"context_line":"            sub_broker.merge_syncs(syncs, incoming)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_8edec63b","line":1857,"range":{"start_line":1855,"start_character":55,"end_line":1857,"end_character":33},"updated":"2018-05-15 23:27:29.000000000","message":"This isn\u0027t so accurate any more, though.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1852,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1853,"context_line":""},{"line_number":1854,"context_line":"        # We also need to sync the sync tables as we have no idea how long"},{"line_number":1855,"context_line":"        # sharding will take and we want to be able to continue replication"},{"line_number":1856,"context_line":"        # (albeit we only ever usync in sharding state, but we need to keep"},{"line_number":1857,"context_line":"        # the sync point mutable)"},{"line_number":1858,"context_line":"        for incoming in (True, False):"},{"line_number":1859,"context_line":"            syncs \u003d self.get_syncs(incoming)"},{"line_number":1860,"context_line":"            sub_broker.merge_syncs(syncs, incoming)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_97a9ed24","line":1857,"range":{"start_line":1855,"start_character":55,"end_line":1857,"end_character":33},"in_reply_to":"5f7c97a3_8edec63b","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1868,"context_line":"                    \"values (?, \u0027pivted_remove\u0027, ?, 0, \u0027\u0027, ?)\""},{"line_number":1869,"context_line":"                conn.execute(sql, (max_row, Timestamp.now().internal,"},{"line_number":1870,"context_line":"                                   MD5_OF_EMPTY_STRING))"},{"line_number":1871,"context_line":"                conn.execute(\u0027DELETE FROM object WHERE ROWID \u003d %d\u0027 % max_row)"},{"line_number":1872,"context_line":"                conn.commit()"},{"line_number":1873,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1874,"context_line":"                self.logger.error(\u0027Failed to set the ROWID of the shard range \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_2ee9b2e4","line":1871,"range":{"start_line":1871,"start_character":63,"end_line":1871,"end_character":65},"updated":"2018-05-15 23:27:29.000000000","message":"Seems weird to do SQL params above, but string interpolation here... *shrug*","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1868,"context_line":"                    \"values (?, \u0027pivted_remove\u0027, ?, 0, \u0027\u0027, ?)\""},{"line_number":1869,"context_line":"                conn.execute(sql, (max_row, Timestamp.now().internal,"},{"line_number":1870,"context_line":"                                   MD5_OF_EMPTY_STRING))"},{"line_number":1871,"context_line":"                conn.execute(\u0027DELETE FROM object WHERE ROWID \u003d %d\u0027 % max_row)"},{"line_number":1872,"context_line":"                conn.commit()"},{"line_number":1873,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1874,"context_line":"                self.logger.error(\u0027Failed to set the ROWID of the shard range \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_08adaae9","line":1871,"range":{"start_line":1871,"start_character":63,"end_line":1871,"end_character":65},"in_reply_to":"5f7c97a3_2ee9b2e4","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1872,"context_line":"                conn.commit()"},{"line_number":1873,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1874,"context_line":"                self.logger.error(\u0027Failed to set the ROWID of the shard range \u0027"},{"line_number":1875,"context_line":"                                  \u0027database for %s/%s: %s\u0027, self.account,"},{"line_number":1876,"context_line":"                                  self.container, err)"},{"line_number":1877,"context_line":""},{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_4ee42e0d","line":1876,"range":{"start_line":1875,"start_character":60,"end_line":1876,"end_character":48},"updated":"2018-05-15 23:27:29.000000000","message":"nit: could just use self.path now.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1872,"context_line":"                conn.commit()"},{"line_number":1873,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1874,"context_line":"                self.logger.error(\u0027Failed to set the ROWID of the shard range \u0027"},{"line_number":1875,"context_line":"                                  \u0027database for %s/%s: %s\u0027, self.account,"},{"line_number":1876,"context_line":"                                  self.container, err)"},{"line_number":1877,"context_line":""},{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_f2d94f82","line":1876,"range":{"start_line":1875,"start_character":60,"end_line":1876,"end_character":48},"in_reply_to":"5f7c97a3_4ee42e0d","updated":"2018-05-16 18:49:30.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7b2ca946ac931a5da628bd3be57ee9fb374b7357","unresolved":false,"context_lines":[{"line_number":1874,"context_line":"                self.logger.error(\u0027Failed to set the ROWID of the shard range \u0027"},{"line_number":1875,"context_line":"                                  \u0027database for %s/%s: %s\u0027, self.account,"},{"line_number":1876,"context_line":"                                  self.container, err)"},{"line_number":1877,"context_line":""},{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"},{"line_number":1880,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_92e49b3d","line":1877,"updated":"2018-05-16 18:49:30.000000000","message":"return False","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1875,"context_line":"                                  \u0027database for %s/%s: %s\u0027, self.account,"},{"line_number":1876,"context_line":"                                  self.container, err)"},{"line_number":1877,"context_line":""},{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"},{"line_number":1880,"context_line":"            try:"},{"line_number":1881,"context_line":"                data \u003d self.conn.execute(\u0027SELECT hash, created_at \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_eeee5aeb","line":1878,"range":{"start_line":1878,"start_character":40,"end_line":1878,"end_character":44},"updated":"2018-05-15 23:27:29.000000000","message":"Since we\u0027re no longer going to replicate directly to the epoch db... is there any point in setting the hash now? Probably better to let it reflect the actual object rows in the db, yeah? Otherwise, I can\u0027t imagine two replicas of a container that have shrunk back to having their epoch dbs be responsible for objects actually getting to a matching hash -- plus we\u0027d lose the ability to check who got it *right* because they started from different (and unknown) places.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1875,"context_line":"                                  \u0027database for %s/%s: %s\u0027, self.account,"},{"line_number":1876,"context_line":"                                  self.container, err)"},{"line_number":1877,"context_line":""},{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"},{"line_number":1880,"context_line":"            try:"},{"line_number":1881,"context_line":"                data \u003d self.conn.execute(\u0027SELECT hash, created_at \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_63a801db","line":1878,"range":{"start_line":1878,"start_character":40,"end_line":1878,"end_character":44},"in_reply_to":"5f7c97a3_eeee5aeb","updated":"2018-05-17 16:42:46.000000000","message":"Yes I think I agree, if the retiring db hashes are out of sync they will never get back to sync once we stop replicating rows between them.\n\nI\u0027m not convinced we even need to set max row ids in the fresh db but I\u0027m wary of tinkering with that at this late stage.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"},{"line_number":1880,"context_line":"            try:"},{"line_number":1881,"context_line":"                data \u003d self.conn.execute(\u0027SELECT hash, created_at \u0027"},{"line_number":1882,"context_line":"                                         \u0027FROM container_stat\u0027)"},{"line_number":1883,"context_line":"                data.row_factory \u003d None"},{"line_number":1884,"context_line":"                old_hash, created_at \u003d data.fetchone()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_cec11e56","line":1881,"range":{"start_line":1881,"start_character":49,"end_line":1881,"end_character":65},"updated":"2018-05-15 23:27:29.000000000","message":"Aren\u0027t these already available in `info`? Are we concerned about races or something? Wouldn\u0027t that *still* be a concern since we aren\u0027t doing any sort of locking?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1878,"context_line":"            # set the the create_at and hash in the container_info table the"},{"line_number":1879,"context_line":"            # same in both brokers"},{"line_number":1880,"context_line":"            try:"},{"line_number":1881,"context_line":"                data \u003d self.conn.execute(\u0027SELECT hash, created_at \u0027"},{"line_number":1882,"context_line":"                                         \u0027FROM container_stat\u0027)"},{"line_number":1883,"context_line":"                data.row_factory \u003d None"},{"line_number":1884,"context_line":"                old_hash, created_at \u003d data.fetchone()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_a33139a5","line":1881,"range":{"start_line":1881,"start_character":49,"end_line":1881,"end_character":65},"in_reply_to":"5f7c97a3_cec11e56","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1898,"context_line":""},{"line_number":1899,"context_line":"        # Now we need to reset the connection so next time the correct database"},{"line_number":1900,"context_line":"        # will be in use."},{"line_number":1901,"context_line":"        self.conn \u003d None"},{"line_number":1902,"context_line":"        self.reload_db_files()"},{"line_number":1903,"context_line":""},{"line_number":1904,"context_line":"        return True"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_2ed21229","line":1901,"updated":"2018-05-15 23:27:29.000000000","message":"I wonder if we should maybe move this into reload_db_files... http://paste.openstack.org/show/721051/ seems not-crazy.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1898,"context_line":""},{"line_number":1899,"context_line":"        # Now we need to reset the connection so next time the correct database"},{"line_number":1900,"context_line":"        # will be in use."},{"line_number":1901,"context_line":"        self.conn \u003d None"},{"line_number":1902,"context_line":"        self.reload_db_files()"},{"line_number":1903,"context_line":""},{"line_number":1904,"context_line":"        return True"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_63032175","line":1901,"in_reply_to":"5f7c97a3_2ed21229","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1921,"context_line":"        if len(self.db_files) \u003c 2:"},{"line_number":1922,"context_line":"            self.logger.warning("},{"line_number":1923,"context_line":"                \u0027Refusing to delete db file %r: no fresher db file.\u0027,"},{"line_number":1924,"context_line":"                self.db_files)"},{"line_number":1925,"context_line":"            return False"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_8eac869f","line":1924,"range":{"start_line":1924,"start_character":16,"end_line":1924,"end_character":29},"updated":"2018-05-15 23:27:29.000000000","message":"self.db_files[0]? *Hopefully* there\u0027d be at least one...","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1921,"context_line":"        if len(self.db_files) \u003c 2:"},{"line_number":1922,"context_line":"            self.logger.warning("},{"line_number":1923,"context_line":"                \u0027Refusing to delete db file %r: no fresher db file.\u0027,"},{"line_number":1924,"context_line":"                self.db_files)"},{"line_number":1925,"context_line":"            return False"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_83299df3","line":1924,"range":{"start_line":1924,"start_character":16,"end_line":1924,"end_character":29},"in_reply_to":"5f7c97a3_8eac869f","updated":"2018-05-17 16:42:46.000000000","message":"Yes, but I see no harm in allowing for other possibilities","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1925,"context_line":"            return False"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"        try:"},{"line_number":1928,"context_line":"            os.unlink(self.db_files[0])"},{"line_number":1929,"context_line":"        except OSError as err:"},{"line_number":1930,"context_line":"            if err.errno !\u003d errno.ENOENT:"},{"line_number":1931,"context_line":"                self.logger.exception(\u0027Failed to unlink %s\u0027 % self._db_file)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_2e97724a","line":1928,"range":{"start_line":1928,"start_character":35,"end_line":1928,"end_character":38},"updated":"2018-05-15 23:27:29.000000000","message":"So here in set_sharded_state we unlink the *oldest*... but over in get_brokers we make available the *second-newest*? That makes me worry a bit... Should this be [-2]?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1925,"context_line":"            return False"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"        try:"},{"line_number":1928,"context_line":"            os.unlink(self.db_files[0])"},{"line_number":1929,"context_line":"        except OSError as err:"},{"line_number":1930,"context_line":"            if err.errno !\u003d errno.ENOENT:"},{"line_number":1931,"context_line":"                self.logger.exception(\u0027Failed to unlink %s\u0027 % self._db_file)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_036b4d1f","line":1928,"range":{"start_line":1928,"start_character":35,"end_line":1928,"end_character":38},"in_reply_to":"5f7c97a3_2e97724a","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1935,"context_line":"        self.conn \u003d None"},{"line_number":1936,"context_line":"        self.reload_db_files()"},{"line_number":1937,"context_line":""},{"line_number":1938,"context_line":"        return True"},{"line_number":1939,"context_line":""},{"line_number":1940,"context_line":"    def get_brokers(self):"},{"line_number":1941,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0e987672","line":1938,"updated":"2018-05-15 23:27:29.000000000","message":"So if we somehow found ourselves with three or more db_files... we\u0027d delete the oldest one... and still be in state sharding, yeah? Should we maybe have this last bit be\n\n if len(self.db_files) \u003e\u003d 2:\n     self.logger.warning(\u0027Still have multiple db files \u0027\n                         \u0027after unlinking: %r\u0027, self.db_files)\n     return False\n return True\n\n?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1935,"context_line":"        self.conn \u003d None"},{"line_number":1936,"context_line":"        self.reload_db_files()"},{"line_number":1937,"context_line":""},{"line_number":1938,"context_line":"        return True"},{"line_number":1939,"context_line":""},{"line_number":1940,"context_line":"    def get_brokers(self):"},{"line_number":1941,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_4312a564","line":1938,"in_reply_to":"5f7c97a3_0e987672","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1955,"context_line":"                force_db_file\u003dTrue, skip_commits\u003dTrue)"},{"line_number":1956,"context_line":"            for db_file in self.db_files[-2:-1]]"},{"line_number":1957,"context_line":"        brokers.append(self)"},{"line_number":1958,"context_line":"        return brokers"},{"line_number":1959,"context_line":""},{"line_number":1960,"context_line":"    def set_sharding_sysmeta(self, key, value):"},{"line_number":1961,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ce8fde27","line":1958,"updated":"2018-05-15 23:27:29.000000000","message":"OK, so\n\n \u003e\u003e\u003e c \u003d ContainerBroker(\u0027/mnt/swift-disk/sdb4/containers/15/cae/0f65ef68c2db3f0768374d15c9a14cae/0f65ef68c2db3f0768374d15c9a14cae_1526330871.38137.db\u0027)\n \u003e\u003e\u003e [x.db_file for x in c.get_brokers()]\n [\u0027/mnt/swift-disk/sdb4/containers/15/cae/0f65ef68c2db3f0768374d15c9a14cae/0f65ef68c2db3f0768374d15c9a14cae.db\u0027,\n  \u0027/mnt/swift-disk/sdb4/containers/15/cae/0f65ef68c2db3f0768374d15c9a14cae/0f65ef68c2db3f0768374d15c9a14cae_1526330871.38137.db\u0027]\n\nCheck.\n\n \u003e\u003e\u003e [x.db_file for x in c.get_brokers()[0].get_brokers()]\n [\u0027/mnt/swift-disk/sdb4/containers/15/cae/0f65ef68c2db3f0768374d15c9a14cae/0f65ef68c2db3f0768374d15c9a14cae.db\u0027,\n  \u0027/mnt/swift-disk/sdb4/containers/15/cae/0f65ef68c2db3f0768374d15c9a14cae/0f65ef68c2db3f0768374d15c9a14cae.db\u0027]\n\nHmm....\n\nMaybe have a\n\n if self.force_db_file:\n     return [self]\n\nfairly early on?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1955,"context_line":"                force_db_file\u003dTrue, skip_commits\u003dTrue)"},{"line_number":1956,"context_line":"            for db_file in self.db_files[-2:-1]]"},{"line_number":1957,"context_line":"        brokers.append(self)"},{"line_number":1958,"context_line":"        return brokers"},{"line_number":1959,"context_line":""},{"line_number":1960,"context_line":"    def set_sharding_sysmeta(self, key, value):"},{"line_number":1961,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_c3c1d4e6","line":1958,"in_reply_to":"5f7c97a3_ce8fde27","updated":"2018-05-17 16:42:46.000000000","message":"good catch, although I\u0027d rather make the method always return two brokers when there are two dbs.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1985,"context_line":"            return info.get(key)"},{"line_number":1986,"context_line":"        return info"},{"line_number":1987,"context_line":""},{"line_number":1988,"context_line":"    def get_items_since(self, start, count, include_sharding\u003dFalse):"},{"line_number":1989,"context_line":"        \"\"\""},{"line_number":1990,"context_line":"        Get a list of objects in the database between start and end."},{"line_number":1991,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6e824a02","line":1988,"updated":"2018-05-15 23:27:29.000000000","message":"Since we defer replication, I don\u0027t think this override is necessary any more. http://paste.openstack.org/show/721052/","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1985,"context_line":"            return info.get(key)"},{"line_number":1986,"context_line":"        return info"},{"line_number":1987,"context_line":""},{"line_number":1988,"context_line":"    def get_items_since(self, start, count, include_sharding\u003dFalse):"},{"line_number":1989,"context_line":"        \"\"\""},{"line_number":1990,"context_line":"        Get a list of objects in the database between start and end."},{"line_number":1991,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_460e3233","line":1988,"in_reply_to":"5f7c97a3_6e824a02","updated":"2018-05-17 16:42:46.000000000","message":"agree, and I thought I had a TODO to revisit this :/. Thanks for diff - less code, yay!","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2010,"context_line":"        Attempt to get the root shard container name and account for the"},{"line_number":2011,"context_line":"        container represented by this broker."},{"line_number":2012,"context_line":""},{"line_number":2013,"context_line":"        A container shard has \u0027X-Container-Sysmeta-Shard-{Account,Container}"},{"line_number":2014,"context_line":"        set, which will contain the relevant values for the root shard"},{"line_number":2015,"context_line":"        container. If they don\u0027t exist, then it returns the account and"},{"line_number":2016,"context_line":"        container associated directly with the broker."}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ee75fa32","line":2013,"range":{"start_line":2013,"start_character":30,"end_line":2013,"end_character":76},"updated":"2018-05-15 23:27:29.000000000","message":"X-Container-Sysmeta-Shard-Root, now","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":2010,"context_line":"        Attempt to get the root shard container name and account for the"},{"line_number":2011,"context_line":"        container represented by this broker."},{"line_number":2012,"context_line":""},{"line_number":2013,"context_line":"        A container shard has \u0027X-Container-Sysmeta-Shard-{Account,Container}"},{"line_number":2014,"context_line":"        set, which will contain the relevant values for the root shard"},{"line_number":2015,"context_line":"        container. If they don\u0027t exist, then it returns the account and"},{"line_number":2016,"context_line":"        container associated directly with the broker."}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_e6e41e3f","line":2013,"range":{"start_line":2013,"start_character":30,"end_line":2013,"end_character":76},"in_reply_to":"5f7c97a3_ee75fa32","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2030,"context_line":"        if path.count(\u0027/\u0027) !\u003d 1 or path.strip(\u0027/\u0027).count(\u0027/\u0027) \u003d\u003d 0:"},{"line_number":2031,"context_line":"            raise ValueError(\u0027Expected X-Container-Sysmeta-Shard-Root to be \u0027"},{"line_number":2032,"context_line":"                             \"of the form \u0027account/container\u0027, got %r\" % path)"},{"line_number":2033,"context_line":"        self._root_account, self._root_container \u003d tuple(path.split(\u0027/\u0027))"},{"line_number":2034,"context_line":""},{"line_number":2035,"context_line":"    @property"},{"line_number":2036,"context_line":"    def root_account(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ae914249","line":2033,"updated":"2018-05-15 23:27:29.000000000","message":"Worth pulling this validation and that in ShardRange.name.setter out to some helper in utils?\n\nHell, should we just use split_path(\u0027/\u0027 + path) for both of these?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":2030,"context_line":"        if path.count(\u0027/\u0027) !\u003d 1 or path.strip(\u0027/\u0027).count(\u0027/\u0027) \u003d\u003d 0:"},{"line_number":2031,"context_line":"            raise ValueError(\u0027Expected X-Container-Sysmeta-Shard-Root to be \u0027"},{"line_number":2032,"context_line":"                             \"of the form \u0027account/container\u0027, got %r\" % path)"},{"line_number":2033,"context_line":"        self._root_account, self._root_container \u003d tuple(path.split(\u0027/\u0027))"},{"line_number":2034,"context_line":""},{"line_number":2035,"context_line":"    @property"},{"line_number":2036,"context_line":"    def root_account(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_069a7a6d","line":2033,"in_reply_to":"5f7c97a3_ae914249","updated":"2018-05-17 16:42:46.000000000","message":"+1 use split_path","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2079,"context_line":"                args.append(str(last_upper))"},{"line_number":2080,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % (shard_size - 1)"},{"line_number":2081,"context_line":"            row \u003d connection.execute(sql, args).fetchone()"},{"line_number":2082,"context_line":"            return row[\u0027name\u0027] if row else None"},{"line_number":2083,"context_line":""},{"line_number":2084,"context_line":"    def find_shard_ranges(self, shard_size, limit\u003d-1, existing_ranges\u003dNone):"},{"line_number":2085,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6e97aa3b","line":2082,"updated":"2018-05-15 23:27:29.000000000","message":"Good, EXPLAIN QUERY PLAN says just\n\n\u003e SEARCH TABLE object USING COVERING INDEX ix_object_deleted_name (deleted\u003d?)\n\nor\n\n\u003e SEARCH TABLE object USING COVERING INDEX ix_object_deleted_name (deleted\u003d? AND name\u003e?)\n\nand nothing about temp b-trees.\n\nDo we need to worry about deleted\u003d0 vs. deleted\u003d+0 for pre-name-index DBs? (Man, those are gonna hurt... but at least the epoch db and shards will finally get an index?)","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":2079,"context_line":"                args.append(str(last_upper))"},{"line_number":2080,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % (shard_size - 1)"},{"line_number":2081,"context_line":"            row \u003d connection.execute(sql, args).fetchone()"},{"line_number":2082,"context_line":"            return row[\u0027name\u0027] if row else None"},{"line_number":2083,"context_line":""},{"line_number":2084,"context_line":"    def find_shard_ranges(self, shard_size, limit\u003d-1, existing_ranges\u003dNone):"},{"line_number":2085,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_86d44abf","line":2082,"in_reply_to":"5f7c97a3_6e97aa3b","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2123,"context_line":"            else:"},{"line_number":2124,"context_line":"                # else: object count in existing shard ranges may have changed"},{"line_number":2125,"context_line":"                # since they were found so progress cannot be reliably"},{"line_number":2126,"context_line":"                # calculated; use default progress pf zero - that\u0027s ok,"},{"line_number":2127,"context_line":"                # progress is used for optimisation not correctness"},{"line_number":2128,"context_line":"                progress_reliable \u003d False"},{"line_number":2129,"context_line":"            last_shard_upper \u003d existing_ranges[-1].upper"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_2ea532a2","line":2126,"range":{"start_line":2126,"start_character":51,"end_line":2126,"end_character":53},"updated":"2018-05-15 23:27:29.000000000","message":"of","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2139,"context_line":"        sub_broker \u003d self.get_brokers()[0]"},{"line_number":2140,"context_line":"        index \u003d len(existing_ranges)"},{"line_number":2141,"context_line":"        while limit \u003c 0 or len(found_ranges) \u003c limit:"},{"line_number":2142,"context_line":"            if progress + shard_size \u003e\u003d object_count:"},{"line_number":2143,"context_line":"                # next shard point is at or beyond final object name so don\u0027t"},{"line_number":2144,"context_line":"                # bother with db query"},{"line_number":2145,"context_line":"                next_shard_upper \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_4ea0ae90","line":2142,"updated":"2018-05-15 23:27:29.000000000","message":"Should this be\n\n if progress_reliable and progress + shard_size \u003e\u003d object_count:\n\n?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":2139,"context_line":"        sub_broker \u003d self.get_brokers()[0]"},{"line_number":2140,"context_line":"        index \u003d len(existing_ranges)"},{"line_number":2141,"context_line":"        while limit \u003c 0 or len(found_ranges) \u003c limit:"},{"line_number":2142,"context_line":"            if progress + shard_size \u003e\u003d object_count:"},{"line_number":2143,"context_line":"                # next shard point is at or beyond final object name so don\u0027t"},{"line_number":2144,"context_line":"                # bother with db query"},{"line_number":2145,"context_line":"                next_shard_upper \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_e9c38f37","line":2142,"in_reply_to":"5f7c97a3_4ea0ae90","updated":"2018-05-17 16:42:46.000000000","message":"progress_reliable False implies that progress started at 0 so this condition is still valid\n\nFWIW, as a follow up I might replace this with a ScanningContext similar to a CleavingContext","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2147,"context_line":"                try:"},{"line_number":2148,"context_line":"                    next_shard_upper \u003d sub_broker._get_next_shard_range_upper("},{"line_number":2149,"context_line":"                        shard_size, last_shard_upper)"},{"line_number":2150,"context_line":"                except sqlite3.OperationalError as err:"},{"line_number":2151,"context_line":"                    self.logger.exception(\"Problem finding shard point: \", err)"},{"line_number":2152,"context_line":"                    break"},{"line_number":2153,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_eeaada6e","line":2150,"updated":"2018-05-15 23:27:29.000000000","message":"Worth catching LockTimeouts?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":2147,"context_line":"                try:"},{"line_number":2148,"context_line":"                    next_shard_upper \u003d sub_broker._get_next_shard_range_upper("},{"line_number":2149,"context_line":"                        shard_size, last_shard_upper)"},{"line_number":2150,"context_line":"                except sqlite3.OperationalError as err:"},{"line_number":2151,"context_line":"                    self.logger.exception(\"Problem finding shard point: \", err)"},{"line_number":2152,"context_line":"                    break"},{"line_number":2153,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0c5059d4","line":2150,"in_reply_to":"5f7c97a3_eeaada6e","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":602,"context_line":"    def _get_deleted_key(self, connection):"},{"line_number":603,"context_line":"        if self.get_db_version(connection) \u003c 1:"},{"line_number":604,"context_line":"            return \u0027+deleted\u0027"},{"line_number":605,"context_line":"        return \u0027deleted\u0027"},{"line_number":606,"context_line":""},{"line_number":607,"context_line":"    def _newid(self, conn):"},{"line_number":608,"context_line":"        conn.execute(\u0027\u0027\u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_b1b655dc","line":605,"updated":"2018-05-17 22:46:22.000000000","message":"May as well make our cargo-culting more convenient ;-)","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":602,"context_line":"    def _get_deleted_key(self, connection):"},{"line_number":603,"context_line":"        if self.get_db_version(connection) \u003c 1:"},{"line_number":604,"context_line":"            return \u0027+deleted\u0027"},{"line_number":605,"context_line":"        return \u0027deleted\u0027"},{"line_number":606,"context_line":""},{"line_number":607,"context_line":"    def _newid(self, conn):"},{"line_number":608,"context_line":"        conn.execute(\u0027\u0027\u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_ea512164","line":605,"in_reply_to":"5f7c97a3_b1b655dc","updated":"2018-05-18 17:56:24.000000000","message":"and more obvious :)","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1150,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1151,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1152,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1153,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1154,"context_line":"                        curs.close()"},{"line_number":1155,"context_line":"                        break"},{"line_number":1156,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_317c851f","line":1153,"updated":"2018-05-17 22:46:22.000000000","message":"Even if we\u0027re *not* going to fix up the number of columns for subdirs... should we at least make this consistently call transform_func? This is the *only place* that doesn\u0027t use it...","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1150,"context_line":"                            delim_force_gte \u003d True"},{"line_number":1151,"context_line":"                        dir_name \u003d name[:end + 1]"},{"line_number":1152,"context_line":"                        if dir_name !\u003d orig_marker:"},{"line_number":1153,"context_line":"                            results.append([dir_name, \u00270\u0027, 0, None, \u0027\u0027])"},{"line_number":1154,"context_line":"                        curs.close()"},{"line_number":1155,"context_line":"                        break"},{"line_number":1156,"context_line":"                    results.append(transform_func(row))"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_0ad01dc9","line":1153,"in_reply_to":"5f7c97a3_317c851f","updated":"2018-05-18 17:56:24.000000000","message":"follow up on master","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1651,"context_line":""},{"line_number":1652,"context_line":"    def get_shard_ranges(self, marker\u003dNone, end_marker\u003dNone, includes\u003dNone,"},{"line_number":1653,"context_line":"                         reverse\u003dFalse, include_deleted\u003dFalse, states\u003dNone,"},{"line_number":1654,"context_line":"                         exclude_states\u003dNone, include_own\u003dFalse,"},{"line_number":1655,"context_line":"                         exclude_others\u003dFalse, fill_gaps\u003dFalse):"},{"line_number":1656,"context_line":"        \"\"\""},{"line_number":1657,"context_line":"        Returns a list of persisted shard ranges."}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_f1460dc1","line":1654,"range":{"start_line":1654,"start_character":25,"end_line":1654,"end_character":39},"updated":"2018-05-17 22:46:22.000000000","message":"I don\u0027t think this is ever actually used...","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1651,"context_line":""},{"line_number":1652,"context_line":"    def get_shard_ranges(self, marker\u003dNone, end_marker\u003dNone, includes\u003dNone,"},{"line_number":1653,"context_line":"                         reverse\u003dFalse, include_deleted\u003dFalse, states\u003dNone,"},{"line_number":1654,"context_line":"                         exclude_states\u003dNone, include_own\u003dFalse,"},{"line_number":1655,"context_line":"                         exclude_others\u003dFalse, fill_gaps\u003dFalse):"},{"line_number":1656,"context_line":"        \"\"\""},{"line_number":1657,"context_line":"        Returns a list of persisted shard ranges."}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_2ad5d9b9","line":1654,"range":{"start_line":1654,"start_character":25,"end_line":1654,"end_character":39},"in_reply_to":"5f7c97a3_f1460dc1","updated":"2018-05-18 17:56:24.000000000","message":"ugh. fix on master?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1704,"context_line":"        # note if this ever changes to *not* sort by upper first then it breaks"},{"line_number":1705,"context_line":"        # a key assumption for bisect, which is used by utils.find_shard_ranges"},{"line_number":1706,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1707,"context_line":"        if includes:"},{"line_number":1708,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1709,"context_line":"            return [shard_range] if shard_range else []"},{"line_number":1710,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_915f9971","line":1707,"updated":"2018-05-17 22:46:22.000000000","message":"Might be worth plumbing down into _get_shard_range_rows -- should be able to translate to\n\n WHERE upper \u003e\u003d includes AND lower \u003c includes\n\nin SQL?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1704,"context_line":"        # note if this ever changes to *not* sort by upper first then it breaks"},{"line_number":1705,"context_line":"        # a key assumption for bisect, which is used by utils.find_shard_ranges"},{"line_number":1706,"context_line":"        shard_ranges.sort(key\u003dlambda sr: (sr.upper, sr.state, sr.lower))"},{"line_number":1707,"context_line":"        if includes:"},{"line_number":1708,"context_line":"            shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":1709,"context_line":"            return [shard_range] if shard_range else []"},{"line_number":1710,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_ead601c4","line":1707,"in_reply_to":"5f7c97a3_915f9971","updated":"2018-05-18 17:56:24.000000000","message":"fix on master","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1846,"context_line":"        # copy relevant data from the retiring db to the fresh db"},{"line_number":1847,"context_line":"        fresh_broker.update_metadata(self.metadata)"},{"line_number":1848,"context_line":"        fresh_broker.merge_shard_ranges("},{"line_number":1849,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1850,"context_line":"        # copy sync points so that any peer in sync with retiring db will"},{"line_number":1851,"context_line":"        # appear to be in sync with the fresh db, although the peer shouldn\u0027t"},{"line_number":1852,"context_line":"        # attempt to replicate objects to a db with shard ranges."}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_f6192322","line":1849,"range":{"start_line":1849,"start_character":17,"end_line":1849,"end_character":33},"updated":"2018-05-17 22:46:22.000000000","message":"Worth using get_all_shard_range_data?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1846,"context_line":"        # copy relevant data from the retiring db to the fresh db"},{"line_number":1847,"context_line":"        fresh_broker.update_metadata(self.metadata)"},{"line_number":1848,"context_line":"        fresh_broker.merge_shard_ranges("},{"line_number":1849,"context_line":"            self.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue))"},{"line_number":1850,"context_line":"        # copy sync points so that any peer in sync with retiring db will"},{"line_number":1851,"context_line":"        # appear to be in sync with the fresh db, although the peer shouldn\u0027t"},{"line_number":1852,"context_line":"        # attempt to replicate objects to a db with shard ranges."}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_eaffe131","line":1849,"range":{"start_line":1849,"start_character":17,"end_line":1849,"end_character":33},"in_reply_to":"5f7c97a3_f6192322","updated":"2018-05-18 17:56:24.000000000","message":"Done","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1871,"context_line":"                self.logger.error("},{"line_number":1872,"context_line":"                    \u0027Failed to set the ROWID of the fresh database for %s: %s\u0027,"},{"line_number":1873,"context_line":"                    self.path, err)"},{"line_number":1874,"context_line":"                return False"},{"line_number":1875,"context_line":""},{"line_number":1876,"context_line":"            # Set the the created_at and hash in the container_info table the"},{"line_number":1877,"context_line":"            # same in both brokers"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_36cebb83","line":1874,"updated":"2018-05-17 22:46:22.000000000","message":"Good catch!","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1873,"context_line":"                    self.path, err)"},{"line_number":1874,"context_line":"                return False"},{"line_number":1875,"context_line":""},{"line_number":1876,"context_line":"            # Set the the created_at and hash in the container_info table the"},{"line_number":1877,"context_line":"            # same in both brokers"},{"line_number":1878,"context_line":"            try:"},{"line_number":1879,"context_line":"                fresh_broker_conn.execute("}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_76a9938e","line":1876,"range":{"start_line":1876,"start_character":18,"end_line":1876,"end_character":45},"updated":"2018-05-17 22:46:22.000000000","message":"the created_at","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1873,"context_line":"                    self.path, err)"},{"line_number":1874,"context_line":"                return False"},{"line_number":1875,"context_line":""},{"line_number":1876,"context_line":"            # Set the the created_at and hash in the container_info table the"},{"line_number":1877,"context_line":"            # same in both brokers"},{"line_number":1878,"context_line":"            try:"},{"line_number":1879,"context_line":"                fresh_broker_conn.execute("}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_aaf9e94d","line":1876,"range":{"start_line":1876,"start_character":18,"end_line":1876,"end_character":45},"in_reply_to":"5f7c97a3_76a9938e","updated":"2018-05-18 17:56:24.000000000","message":"Done","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1953,"context_line":"                self.container, self.pending_timeout, self.stale_reads_ok,"},{"line_number":1954,"context_line":"                force_db_file\u003dTrue, skip_commits\u003dbool(db_files))"},{"line_number":1955,"context_line":"            brokers.append(sub_broker)"},{"line_number":1956,"context_line":"        return brokers"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"    def set_sharding_sysmeta(self, key, value):"},{"line_number":1959,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_d62e67f9","line":1956,"updated":"2018-05-17 22:46:22.000000000","message":"Interesting... so if you had\n\n hash.db\n hash_t1.db\n hash_t2.db\n\nand you call get_brokers on ContainerBroker(\u0027hash.db\u0027, force_db_file\u003dTrue)... you *only* get the other dbs, and nothing about the original?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fcd8b67b905906889e6cd59edae9063421e5f608","unresolved":false,"context_lines":[{"line_number":1953,"context_line":"                self.container, self.pending_timeout, self.stale_reads_ok,"},{"line_number":1954,"context_line":"                force_db_file\u003dTrue, skip_commits\u003dbool(db_files))"},{"line_number":1955,"context_line":"            brokers.append(sub_broker)"},{"line_number":1956,"context_line":"        return brokers"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"    def set_sharding_sysmeta(self, key, value):"},{"line_number":1959,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_4d306f08","line":1956,"in_reply_to":"5f7c97a3_2a7159e1","updated":"2018-05-18 18:45:16.000000000","message":"That\u0027s my expectation, too. I maybe would\u0027ve preferred that we handle the oldest *first* and proceed *forward* but w/e.","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":1953,"context_line":"                self.container, self.pending_timeout, self.stale_reads_ok,"},{"line_number":1954,"context_line":"                force_db_file\u003dTrue, skip_commits\u003dbool(db_files))"},{"line_number":1955,"context_line":"            brokers.append(sub_broker)"},{"line_number":1956,"context_line":"        return brokers"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"    def set_sharding_sysmeta(self, key, value):"},{"line_number":1959,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_2a7159e1","line":1956,"in_reply_to":"5f7c97a3_d62e67f9","updated":"2018-05-18 17:56:24.000000000","message":"yes. we should probaby figure out how we would deal with \u003e2 db files. We\u0027ll at least see plenty of warnings. I guess (but not tested) that once the most recent retiree has sharded the next oldest would become the retiree?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"052617defc24d8f98a68f5278005f602738eeb10","unresolved":false,"context_lines":[{"line_number":860,"context_line":"    def _get_alternate_object_stats(self):"},{"line_number":861,"context_line":"        state \u003d self.get_db_state()"},{"line_number":862,"context_line":"        if state \u003d\u003d SHARDING:"},{"line_number":863,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":864,"context_line":"            stats \u003d {\u0027object_count\u0027: other_info[\u0027object_count\u0027],"},{"line_number":865,"context_line":"                     \u0027bytes_used\u0027: other_info[\u0027bytes_used\u0027]}"},{"line_number":866,"context_line":"        elif state \u003d\u003d SHARDED and self.is_root_container():"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_1ba951de","line":863,"updated":"2018-05-18 19:48:48.000000000","message":"I\u0027ve occasionally seen tracebacks while running probe tests where this raises\n\n DatabaseConnectionError: DB connection error (/srv/node1/.../731a9c4578723ab9e3649b9ed067fcab.db, 0):\n DB doesn\u0027t exist","commit_id":"14af38a8992c55925a7e6ed74eb9ba1b8d360ff1"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7e6391a270284c383ed6355e5945fc1c9055d52f","unresolved":false,"context_lines":[{"line_number":1892,"context_line":"                    self.path, err)"},{"line_number":1893,"context_line":"                return False"},{"line_number":1894,"context_line":""},{"line_number":1895,"context_line":"            # Set the created_at and hash in the container_info table the same"},{"line_number":1896,"context_line":"            # in both brokers"},{"line_number":1897,"context_line":"            try:"},{"line_number":1898,"context_line":"                fresh_broker_conn.execute("}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_701e6471","line":1895,"range":{"start_line":1895,"start_character":33,"end_line":1895,"end_character":41},"updated":"2018-05-18 19:41:16.000000000","message":"nit: outdated","commit_id":"14af38a8992c55925a7e6ed74eb9ba1b8d360ff1"}],"swift/container/replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":107,"context_line":"        # Always replicate shard ranges"},{"line_number":108,"context_line":"        shard_range_success \u003d self._sync_shard_ranges(broker, http, info[\u0027id\u0027])"},{"line_number":109,"context_line":"        shard_ranges \u003d broker.get_shard_ranges()"},{"line_number":110,"context_line":"        # TODO: also check that own shard range state \u003e ACTIVE?"},{"line_number":111,"context_line":"        if shard_ranges:"},{"line_number":112,"context_line":"            self.logger.warning("},{"line_number":113,"context_line":"                \u0027%s is able to shard -- refusing to replicate objects to peer \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_2ece9262","line":110,"updated":"2018-05-15 23:27:29.000000000","message":"Have we thought more about this yet? I guess it\u0027s an issue when we decide to shrink the last shard DB back into the root, and only lasts until that shrink completes?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":107,"context_line":"        # Always replicate shard ranges"},{"line_number":108,"context_line":"        shard_range_success \u003d self._sync_shard_ranges(broker, http, info[\u0027id\u0027])"},{"line_number":109,"context_line":"        shard_ranges \u003d broker.get_shard_ranges()"},{"line_number":110,"context_line":"        # TODO: also check that own shard range state \u003e ACTIVE?"},{"line_number":111,"context_line":"        if shard_ranges:"},{"line_number":112,"context_line":"            self.logger.warning("},{"line_number":113,"context_line":"                \u0027%s is able to shard -- refusing to replicate objects to peer \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_38cb0f73","line":110,"in_reply_to":"5f7c97a3_2ece9262","updated":"2018-05-17 16:42:46.000000000","message":"hmmm, so the issue as I remember it is that a broker might have shard ranges but not have its own shard range set to sharding, in which case it may not shard, ever...in which case maybe we should replicate.\n\nThe condition here , to not replicate, should be very similar to the condition in cleanup_post_replicate - except cleanup can happen if the broker has sharded.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":127,"context_line":"    def _fetch_and_merge_shard_ranges(self, http, broker):"},{"line_number":128,"context_line":"        response \u003d http.replicate(\u0027get_shard_ranges\u0027)"},{"line_number":129,"context_line":"        if is_success(response.status):"},{"line_number":130,"context_line":"            broker.merge_shard_ranges(json.loads(response.data))"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    def find_local_handoff_for_part(self, part):"},{"line_number":133,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_aedf020b","line":130,"updated":"2018-05-15 23:27:29.000000000","message":"...and we consciously *don\u0027t* worry about\n\n not is_success(response.status)\n\n...because of upgrades, yeah?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":259,"context_line":"            # do any cleanup so we can continue cleaving - this is still"},{"line_number":260,"context_line":"            # considered \"success\""},{"line_number":261,"context_line":"            reason \u003d \u0027requires sharding, state %s\u0027 % broker.get_db_state()"},{"line_number":262,"context_line":"            self.logger.debug(debug_template, reason)"},{"line_number":263,"context_line":"            return True"},{"line_number":264,"context_line":"        return super(ContainerReplicator, self).cleanup_post_replicate("},{"line_number":265,"context_line":"            broker, orig_info, responses)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6ee56adb","line":262,"updated":"2018-05-15 23:27:29.000000000","message":"This feels weird; why not just\n\n self.logger.debug(\u0027Not deleting db %s (requires sharding, state %s)\u0027,\n                   broker.db_file, broker.get_db_state())\n\n?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":259,"context_line":"            # do any cleanup so we can continue cleaving - this is still"},{"line_number":260,"context_line":"            # considered \"success\""},{"line_number":261,"context_line":"            reason \u003d \u0027requires sharding, state %s\u0027 % broker.get_db_state()"},{"line_number":262,"context_line":"            self.logger.debug(debug_template, reason)"},{"line_number":263,"context_line":"            return True"},{"line_number":264,"context_line":"        return super(ContainerReplicator, self).cleanup_post_replicate("},{"line_number":265,"context_line":"            broker, orig_info, responses)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_78a2e74d","line":262,"in_reply_to":"5f7c97a3_6ee56adb","updated":"2018-05-17 16:42:46.000000000","message":"follow on?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":323,"context_line":"            return False"},{"line_number":324,"context_line":""},{"line_number":325,"context_line":"        return super(ContainerReplicator, self)._in_sync("},{"line_number":326,"context_line":"            rinfo, info, broker, local_sync)"},{"line_number":327,"context_line":""},{"line_number":328,"context_line":""},{"line_number":329,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_eed89a1e","line":326,"updated":"2018-05-15 23:27:29.000000000","message":"We could probably avoid this by moving the call to _in_sync into _choose_replication_mode -- then we\u0027d\n\n- fetch remote shard ranges in ContainerReplicator._handle_sync_response\n- call DatabaseReplicator._handle_sync_response\n- push our shard ranges in ContainerReplicator._choose_replication_mode\n- abort before we ever get to calling _in_sync","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":323,"context_line":"            return False"},{"line_number":324,"context_line":""},{"line_number":325,"context_line":"        return super(ContainerReplicator, self)._in_sync("},{"line_number":326,"context_line":"            rinfo, info, broker, local_sync)"},{"line_number":327,"context_line":""},{"line_number":328,"context_line":""},{"line_number":329,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6cb46df3","line":326,"in_reply_to":"5f7c97a3_eed89a1e","updated":"2018-05-17 16:42:46.000000000","message":"follow on?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":329,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"    def _db_file_exists(self, db_path):"},{"line_number":332,"context_line":"        return bool(db_replicator.get_db_files(db_path))"},{"line_number":333,"context_line":""},{"line_number":334,"context_line":"    def _parse_sync_args(self, args):"},{"line_number":335,"context_line":"        parent \u003d super(ContainerReplicatorRpc, self)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0ed4f6e7","line":332,"updated":"2018-05-15 23:27:29.000000000","message":"This comes from utils, yeah? Why go through db_replicator?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":329,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"    def _db_file_exists(self, db_path):"},{"line_number":332,"context_line":"        return bool(db_replicator.get_db_files(db_path))"},{"line_number":333,"context_line":""},{"line_number":334,"context_line":"    def _parse_sync_args(self, args):"},{"line_number":335,"context_line":"        parent \u003d super(ContainerReplicatorRpc, self)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ecee3dec","line":332,"in_reply_to":"5f7c97a3_0ed4f6e7","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":372,"context_line":"    def _post_rsync_then_merge_hook(self, existing_broker, new_broker):"},{"line_number":373,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":374,"context_line":"        # limit in the future."},{"line_number":375,"context_line":"        shard_ranges \u003d existing_broker.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":376,"context_line":"                                                        include_own\u003dTrue)"},{"line_number":377,"context_line":"        if shard_ranges:"},{"line_number":378,"context_line":"            new_broker.merge_shard_ranges(shard_ranges)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ee009aad","line":375,"updated":"2018-05-15 23:27:29.000000000","message":"Better to use existing_broker.get_all_shard_range_data?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":372,"context_line":"    def _post_rsync_then_merge_hook(self, existing_broker, new_broker):"},{"line_number":373,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":374,"context_line":"        # limit in the future."},{"line_number":375,"context_line":"        shard_ranges \u003d existing_broker.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":376,"context_line":"                                                        include_own\u003dTrue)"},{"line_number":377,"context_line":"        if shard_ranges:"},{"line_number":378,"context_line":"            new_broker.merge_shard_ranges(shard_ranges)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_57201e9c","line":375,"in_reply_to":"5f7c97a3_ee009aad","updated":"2018-05-17 16:42:46.000000000","message":"!@?!@! yes! that\u0027s why I wrote the method, how did that get lost?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":382,"context_line":"        return HTTPAccepted()"},{"line_number":383,"context_line":""},{"line_number":384,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":385,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":386,"context_line":"                                               include_own\u003dTrue)"},{"line_number":387,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":388,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ce335eaa","line":385,"updated":"2018-05-15 23:27:29.000000000","message":"ditto","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":382,"context_line":"        return HTTPAccepted()"},{"line_number":383,"context_line":""},{"line_number":384,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":385,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":386,"context_line":"                                               include_own\u003dTrue)"},{"line_number":387,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":388,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_3737a2e6","line":385,"in_reply_to":"5f7c97a3_ce335eaa","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":385,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue,"},{"line_number":386,"context_line":"                                               include_own\u003dTrue)"},{"line_number":387,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":388,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ae086291","line":388,"updated":"2018-05-15 23:27:29.000000000","message":"+1, I really like having _post_rsync_then_merge_hook, merge_shard_ranges, and get_shard_ranges all grouped together like this.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":92,"context_line":"        shard_range_data \u003d broker.get_all_shard_range_data()"},{"line_number":93,"context_line":"        if shard_range_data:"},{"line_number":94,"context_line":"            if not self._send_replicate_request("},{"line_number":95,"context_line":"                    http, \u0027merge_shard_ranges\u0027, shard_range_data, local_id):"},{"line_number":96,"context_line":"                return False"},{"line_number":97,"context_line":"            self.logger.debug(\u0027%s synced %s shard ranges to %s\u0027,"},{"line_number":98,"context_line":"                              broker.db_file, len(shard_range_data),"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_d6c187e6","line":95,"updated":"2018-05-17 22:46:22.000000000","message":"Oh nice, somehow I\u0027d missed the fact that get_all_shard_range_data made everything dicts...","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"    def _in_sync(self, rinfo, info, broker, local_sync):"},{"line_number":318,"context_line":"        # TODO: don\u0027t always sync shard ranges!"},{"line_number":319,"context_line":"        if broker.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue):"},{"line_number":320,"context_line":"            return False"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"        return super(ContainerReplicator, self)._in_sync("}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_d198b11c","line":319,"updated":"2018-05-17 22:46:22.000000000","message":"Maybe use get_all_shard_range_data?","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fcd8b67b905906889e6cd59edae9063421e5f608","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"    def _in_sync(self, rinfo, info, broker, local_sync):"},{"line_number":318,"context_line":"        # TODO: don\u0027t always sync shard ranges!"},{"line_number":319,"context_line":"        if broker.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue):"},{"line_number":320,"context_line":"            return False"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"        return super(ContainerReplicator, self)._in_sync("}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_7003c40e","line":319,"in_reply_to":"5f7c97a3_aa7c89b5","updated":"2018-05-18 18:45:16.000000000","message":"*shrug* w/e -- like I said in patchset 8, I want to get rid of all of this eventually anyway.","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"    def _in_sync(self, rinfo, info, broker, local_sync):"},{"line_number":318,"context_line":"        # TODO: don\u0027t always sync shard ranges!"},{"line_number":319,"context_line":"        if broker.get_shard_ranges(include_own\u003dTrue, include_deleted\u003dTrue):"},{"line_number":320,"context_line":"            return False"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"        return super(ContainerReplicator, self)._in_sync("}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_aa7c89b5","line":319,"in_reply_to":"5f7c97a3_d198b11c","updated":"2018-05-18 17:56:24.000000000","message":"ah, now here I deliberately didn\u0027t to avoid creating a bunch of dicts unnecessarily, but...IDK","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"}],"test/unit/cli/test_info.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":428,"context_line":"            except Exception:"},{"line_number":429,"context_line":"                exp_raised \u003d True"},{"line_number":430,"context_line":"        if exp_raised:"},{"line_number":431,"context_line":"            self.fail(\"Unexpected exception raised\")"},{"line_number":432,"context_line":"        else:"},{"line_number":433,"context_line":"            self.assertGreater(len(out.getvalue().strip()), 800)"},{"line_number":434,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_ce303e59","side":"PARENT","line":431,"updated":"2018-05-15 23:27:29.000000000","message":"+100 wtf were we thinking","commit_id":"e039156e06e7a17754e3acedb67f5af74ea30bdc"}],"test/unit/common/test_db_replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":2212,"context_line":"        rpc \u003d db_replicator.ReplicatorRpc("},{"line_number":2213,"context_line":"            self.root, self.datadir, self.backend, False)"},{"line_number":2214,"context_line":"        response \u003d rpc.dispatch((node[\u0027device\u0027], part, obj_hash),"},{"line_number":2215,"context_line":"                                [\u0027rsync_then_merge\u0027, obj_hash + \u0027.db\u0027, \u0027arg2\u0027])"},{"line_number":2216,"context_line":"        # sanity"},{"line_number":2217,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"},{"line_number":2218,"context_line":"        self.assertEqual(204, response.status_int)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0e707607","side":"PARENT","line":2215,"updated":"2018-05-15 23:27:29.000000000","message":"Still necessary? I kinda remember needing to drop this back on feature/deep, but I don\u0027t remember why now...","commit_id":"e039156e06e7a17754e3acedb67f5af74ea30bdc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1335,"context_line":"                                          mount_check\u003dFalse)"},{"line_number":1336,"context_line":""},{"line_number":1337,"context_line":"        def mock_renamer(old, new):"},{"line_number":1338,"context_line":"            renamer_calls.append((old, new))"},{"line_number":1339,"context_line":""},{"line_number":1340,"context_line":"        self._patch(patch.object, db_replicator, \u0027renamer\u0027, mock_renamer)"},{"line_number":1341,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_6e0c8a93","line":1338,"updated":"2018-05-15 23:27:29.000000000","message":"+1, making assertions in mocks is a terrible idea.","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":1355,"context_line":"        with patch(\u0027swift.common.db_replicator.os\u0027,"},{"line_number":1356,"context_line":"                   new\u003dmock.MagicMock(wraps\u003dos)) as mock_os, \\"},{"line_number":1357,"context_line":"                unit.mock_check_drive(isdir\u003dTrue):"},{"line_number":1358,"context_line":"            mock_os.path.exists.side_effect \u003d [False, True]"},{"line_number":1359,"context_line":"            response \u003d rpc.complete_rsync(\u0027drive\u0027, \u0027/data/db.db\u0027,"},{"line_number":1360,"context_line":"                                          [\u0027arg1\u0027, \u0027arg2\u0027])"},{"line_number":1361,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_8e0fa686","line":1358,"updated":"2018-05-15 23:27:29.000000000","message":"Do we ever test with side_effect\u003d[True, False]?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":1355,"context_line":"        with patch(\u0027swift.common.db_replicator.os\u0027,"},{"line_number":1356,"context_line":"                   new\u003dmock.MagicMock(wraps\u003dos)) as mock_os, \\"},{"line_number":1357,"context_line":"                unit.mock_check_drive(isdir\u003dTrue):"},{"line_number":1358,"context_line":"            mock_os.path.exists.side_effect \u003d [False, True]"},{"line_number":1359,"context_line":"            response \u003d rpc.complete_rsync(\u0027drive\u0027, \u0027/data/db.db\u0027,"},{"line_number":1360,"context_line":"                                          [\u0027arg1\u0027, \u0027arg2\u0027])"},{"line_number":1361,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_11186973","line":1358,"in_reply_to":"5f7c97a3_379ec254","updated":"2018-05-17 22:46:22.000000000","message":"That only covers always-True though -- or am I missing something?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":1355,"context_line":"        with patch(\u0027swift.common.db_replicator.os\u0027,"},{"line_number":1356,"context_line":"                   new\u003dmock.MagicMock(wraps\u003dos)) as mock_os, \\"},{"line_number":1357,"context_line":"                unit.mock_check_drive(isdir\u003dTrue):"},{"line_number":1358,"context_line":"            mock_os.path.exists.side_effect \u003d [False, True]"},{"line_number":1359,"context_line":"            response \u003d rpc.complete_rsync(\u0027drive\u0027, \u0027/data/db.db\u0027,"},{"line_number":1360,"context_line":"                                          [\u0027arg1\u0027, \u0027arg2\u0027])"},{"line_number":1361,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_379ec254","line":1358,"in_reply_to":"5f7c97a3_8e0fa686","updated":"2018-05-17 16:42:46.000000000","message":"test_complete_rsync_db_exists ?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":2257,"context_line":"        rpc \u003d db_replicator.ReplicatorRpc("},{"line_number":2258,"context_line":"            self.root, self.datadir, self.backend, False)"},{"line_number":2259,"context_line":"        response \u003d rpc.dispatch((node[\u0027device\u0027], part, obj_hash),"},{"line_number":2260,"context_line":"                                [\u0027rsync_then_merge\u0027, obj_hash + \u0027.db\u0027])"},{"line_number":2261,"context_line":"        # sanity"},{"line_number":2262,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"},{"line_number":2263,"context_line":"        self.assertEqual(204, response.status_int)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_910c793c","line":2260,"updated":"2018-05-17 22:46:22.000000000","message":"Unnecessary delta.","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":2257,"context_line":"        rpc \u003d db_replicator.ReplicatorRpc("},{"line_number":2258,"context_line":"            self.root, self.datadir, self.backend, False)"},{"line_number":2259,"context_line":"        response \u003d rpc.dispatch((node[\u0027device\u0027], part, obj_hash),"},{"line_number":2260,"context_line":"                                [\u0027rsync_then_merge\u0027, obj_hash + \u0027.db\u0027])"},{"line_number":2261,"context_line":"        # sanity"},{"line_number":2262,"context_line":"        self.assertEqual(\u0027204 No Content\u0027, response.status)"},{"line_number":2263,"context_line":"        self.assertEqual(204, response.status_int)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_ca5ea544","line":2260,"in_reply_to":"5f7c97a3_910c793c","updated":"2018-05-18 17:56:24.000000000","message":"Done","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":3883,"context_line":""},{"line_number":3884,"context_line":"        epoch \u003d utils.Timestamp.now()"},{"line_number":3885,"context_line":"        actual \u003d utils.make_db_file_path(actual, epoch)"},{"line_number":3886,"context_line":"        self.assertEqual(\u0027/path/to/hash_%s.db\u0027 % epoch.internal, actual)"},{"line_number":3887,"context_line":""},{"line_number":3888,"context_line":"        self.assertRaises(ValueError, utils.make_db_file_path,"},{"line_number":3889,"context_line":"                          \u0027/path/to/hash.db\u0027, \u0027bad epoch\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_8ed246d0","line":3886,"updated":"2018-05-15 23:27:29.000000000","message":"Worth having tests where epoch has an offset?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":3883,"context_line":""},{"line_number":3884,"context_line":"        epoch \u003d utils.Timestamp.now()"},{"line_number":3885,"context_line":"        actual \u003d utils.make_db_file_path(actual, epoch)"},{"line_number":3886,"context_line":"        self.assertEqual(\u0027/path/to/hash_%s.db\u0027 % epoch.internal, actual)"},{"line_number":3887,"context_line":""},{"line_number":3888,"context_line":"        self.assertRaises(ValueError, utils.make_db_file_path,"},{"line_number":3889,"context_line":"                          \u0027/path/to/hash.db\u0027, \u0027bad epoch\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_17206649","line":3886,"in_reply_to":"5f7c97a3_8ed246d0","updated":"2018-05-17 16:42:46.000000000","message":"Done","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":4190,"context_line":"        self.assertEqual([], utils.get_db_files(path_1))"},{"line_number":4191,"context_line":"        with open(path_1, \u0027wb\u0027):"},{"line_number":4192,"context_line":"            pass"},{"line_number":4193,"context_line":"        self.assertEqual([path_1], utils.get_db_files(path_1))"},{"line_number":4194,"context_line":""},{"line_number":4195,"context_line":"        path_2 \u003d os.path.join(dbdir, \u0027dbfile_2.db\u0027)"},{"line_number":4196,"context_line":"        self.assertEqual([path_1], utils.get_db_files(path_2))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_aef2a26e","line":4193,"updated":"2018-05-15 23:27:29.000000000","message":"What happens for\n\n utils.get_db_files(dbdir)\n\n?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":4190,"context_line":"        self.assertEqual([], utils.get_db_files(path_1))"},{"line_number":4191,"context_line":"        with open(path_1, \u0027wb\u0027):"},{"line_number":4192,"context_line":"            pass"},{"line_number":4193,"context_line":"        self.assertEqual([path_1], utils.get_db_files(path_1))"},{"line_number":4194,"context_line":""},{"line_number":4195,"context_line":"        path_2 \u003d os.path.join(dbdir, \u0027dbfile_2.db\u0027)"},{"line_number":4196,"context_line":"        self.assertEqual([path_1], utils.get_db_files(path_2))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_b763b2b1","line":4193,"in_reply_to":"5f7c97a3_aef2a26e","updated":"2018-05-17 16:42:46.000000000","message":"empty list","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e4b98c20e8384fd99f7bdc32c5518573250846b7","unresolved":false,"context_lines":[{"line_number":4212,"context_line":"        self.assertEqual([path_1, path_2, path_3], utils.get_db_files(path_3))"},{"line_number":4213,"context_line":""},{"line_number":4214,"context_line":"        other_hash \u003d os.path.join(dbdir, \u0027other.db\u0027)"},{"line_number":4215,"context_line":"        self.assertEqual([], utils.get_db_files(other_hash))"},{"line_number":4216,"context_line":""},{"line_number":4217,"context_line":"        pending \u003d os.path.join(dbdir, \u0027dbfile.pending\u0027)"},{"line_number":4218,"context_line":"        self.assertEqual([path_1, path_2, path_3], utils.get_db_files(pending))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_0ede36d9","line":4215,"updated":"2018-05-15 23:27:29.000000000","message":"Worth creating other.db and verifying that it doesn\u0027t show up in any of the get_db_files calls?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a77d34ac3f22c888c373b0bea47cc7d03450ee73","unresolved":false,"context_lines":[{"line_number":4212,"context_line":"        self.assertEqual([path_1, path_2, path_3], utils.get_db_files(path_3))"},{"line_number":4213,"context_line":""},{"line_number":4214,"context_line":"        other_hash \u003d os.path.join(dbdir, \u0027other.db\u0027)"},{"line_number":4215,"context_line":"        self.assertEqual([], utils.get_db_files(other_hash))"},{"line_number":4216,"context_line":""},{"line_number":4217,"context_line":"        pending \u003d os.path.join(dbdir, \u0027dbfile.pending\u0027)"},{"line_number":4218,"context_line":"        self.assertEqual([path_1, path_2, path_3], utils.get_db_files(pending))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5f7c97a3_57ca5ea1","line":4215,"in_reply_to":"5f7c97a3_0ede36d9","updated":"2018-05-17 16:42:46.000000000","message":"isn\u0027t that same as having the dbfile*.db and checking they dont show up here?","commit_id":"f143b6cca454dbbdfec7582e47810590550cd20c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":3910,"context_line":""},{"line_number":3911,"context_line":"        # epochs shouldn\u0027t have offsets"},{"line_number":3912,"context_line":"        epoch \u003d utils.Timestamp.now()"},{"line_number":3913,"context_line":"        epoch.offset \u003d 10"},{"line_number":3914,"context_line":"        actual \u003d utils.make_db_file_path(actual, epoch)"},{"line_number":3915,"context_line":"        self.assertEqual(\u0027/path/to/hash_%s.db\u0027 % epoch.normal, actual)"},{"line_number":3916,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_16a7ff51","line":3913,"updated":"2018-05-17 22:46:22.000000000","message":"nit: can just do\n\n epoch \u003d utils.Timestamp.now(offset\u003d10)","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":3910,"context_line":""},{"line_number":3911,"context_line":"        # epochs shouldn\u0027t have offsets"},{"line_number":3912,"context_line":"        epoch \u003d utils.Timestamp.now()"},{"line_number":3913,"context_line":"        epoch.offset \u003d 10"},{"line_number":3914,"context_line":"        actual \u003d utils.make_db_file_path(actual, epoch)"},{"line_number":3915,"context_line":"        self.assertEqual(\u0027/path/to/hash_%s.db\u0027 % epoch.normal, actual)"},{"line_number":3916,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_8a58ad56","line":3913,"in_reply_to":"5f7c97a3_16a7ff51","updated":"2018-05-18 17:56:24.000000000","message":"Done","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6757e778aa322b5c4c33628db08ae51a1282119c","unresolved":false,"context_lines":[{"line_number":4213,"context_line":"    @with_tempdir"},{"line_number":4214,"context_line":"    def test_get_db_files(self, tempdir):"},{"line_number":4215,"context_line":"        dbdir \u003d os.path.join(tempdir, \u0027dbdir\u0027)"},{"line_number":4216,"context_line":"        self.assertEqual([], utils.get_db_files(dbdir))"},{"line_number":4217,"context_line":"        path_1 \u003d os.path.join(dbdir, \u0027dbfile.db\u0027)"},{"line_number":4218,"context_line":"        self.assertEqual([], utils.get_db_files(path_1))"},{"line_number":4219,"context_line":"        os.mkdir(dbdir)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_9670efde","line":4216,"updated":"2018-05-17 22:46:22.000000000","message":"This is more interesting when there are actually files in the directory...","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a99cfe58289126a07b07320224267b0615414846","unresolved":false,"context_lines":[{"line_number":4213,"context_line":"    @with_tempdir"},{"line_number":4214,"context_line":"    def test_get_db_files(self, tempdir):"},{"line_number":4215,"context_line":"        dbdir \u003d os.path.join(tempdir, \u0027dbdir\u0027)"},{"line_number":4216,"context_line":"        self.assertEqual([], utils.get_db_files(dbdir))"},{"line_number":4217,"context_line":"        path_1 \u003d os.path.join(dbdir, \u0027dbfile.db\u0027)"},{"line_number":4218,"context_line":"        self.assertEqual([], utils.get_db_files(path_1))"},{"line_number":4219,"context_line":"        os.mkdir(dbdir)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_6a3511f7","line":4216,"in_reply_to":"5f7c97a3_9670efde","updated":"2018-05-18 17:56:24.000000000","message":"Done","commit_id":"b7226b9d87ae86cf30c50e617d93a18bb9cfdb5d"}]}