)]}'
{"doc/saio/swift/container-server/1.conf":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8acbf22737d89ad695a42b0ca14007264c2cf7cc","unresolved":false,"context_lines":[{"line_number":21,"context_line":""},{"line_number":22,"context_line":"[container-replicator]"},{"line_number":23,"context_line":"rsync_module \u003d {replication_ip}::container{replication_port}"},{"line_number":24,"context_line":"per_diff \u003d 100"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"[container-updater]"},{"line_number":27,"context_line":""}],"source_content_type":"text/plain","patch_set":13,"id":"9f6a8fd7_a479f993","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":14},"updated":"2018-04-18 22:00:39.000000000","message":"is this still needed?","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":21,"context_line":""},{"line_number":22,"context_line":"[container-replicator]"},{"line_number":23,"context_line":"rsync_module \u003d {replication_ip}::container{replication_port}"},{"line_number":24,"context_line":"per_diff \u003d 100"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"[container-updater]"},{"line_number":27,"context_line":""}],"source_content_type":"text/plain","patch_set":13,"id":"9f6a8fd7_44849d0c","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":14},"in_reply_to":"9f6a8fd7_8484f55f","updated":"2018-04-19 00:18:42.000000000","message":"Done","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c6c61f78a177fa9d087e423f4c8e70e744c5ad7","unresolved":false,"context_lines":[{"line_number":21,"context_line":""},{"line_number":22,"context_line":"[container-replicator]"},{"line_number":23,"context_line":"rsync_module \u003d {replication_ip}::container{replication_port}"},{"line_number":24,"context_line":"per_diff \u003d 100"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"[container-updater]"},{"line_number":27,"context_line":""}],"source_content_type":"text/plain","patch_set":13,"id":"9f6a8fd7_8484f55f","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":14},"in_reply_to":"9f6a8fd7_a479f993","updated":"2018-04-18 22:03:05.000000000","message":"probably not -- i think?","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"}],"swift/common/db_replicator.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":512,"context_line":"                remote_meta \u003d json.loads(remote_info[\u0027metadata\u0027])"},{"line_number":513,"context_line":"                # Might be nice, but I guess we\u0027re in the wrong object to have debug_timing?"},{"line_number":514,"context_line":"                # with self.debug_timing(\u0027update_metadata\u0027):"},{"line_number":515,"context_line":"                broker.update_metadata(remote_meta)"},{"line_number":516,"context_line":"        elif 200 \u003c\u003d response.status \u003c 300:"},{"line_number":517,"context_line":"            rinfo \u003d json.loads(response.data)"},{"line_number":518,"context_line":"            local_sync \u003d broker.get_sync(rinfo[\u0027id\u0027], incoming\u003dFalse)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_8ed6ab4f","line":515,"range":{"start_line":515,"start_character":16,"end_line":515,"end_character":51},"updated":"2018-03-05 09:53:52.000000000","message":"so was the remote already including metadata in the response body but it was previously never merged into the local?","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1d026b6dd399098f08f7c32c4881abf4867214d2","unresolved":false,"context_lines":[{"line_number":512,"context_line":"                remote_meta \u003d json.loads(remote_info[\u0027metadata\u0027])"},{"line_number":513,"context_line":"                # Might be nice, but I guess we\u0027re in the wrong object to have debug_timing?"},{"line_number":514,"context_line":"                # with self.debug_timing(\u0027update_metadata\u0027):"},{"line_number":515,"context_line":"                broker.update_metadata(remote_meta)"},{"line_number":516,"context_line":"        elif 200 \u003c\u003d response.status \u003c 300:"},{"line_number":517,"context_line":"            rinfo \u003d json.loads(response.data)"},{"line_number":518,"context_line":"            local_sync \u003d broker.get_sync(rinfo[\u0027id\u0027], incoming\u003dFalse)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_fa235ebd","line":515,"range":{"start_line":515,"start_character":16,"end_line":515,"end_character":51},"in_reply_to":"ff6b8bd7_8ed6ab4f","updated":"2018-03-05 17:21:47.000000000","message":"Yup, see https://review.openstack.org/#/c/464084/","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":525,"context_line":"            # more than 50% and the difference is greater than per_diff,"},{"line_number":526,"context_line":"            # rsync then do a remote merge."},{"line_number":527,"context_line":"            # NOTE: difference \u003e per_diff stops us from dropping to rsync"},{"line_number":528,"context_line":"            # on smaller containers, who have only a few rows to sync."},{"line_number":529,"context_line":"            if (rinfo[\u0027max_row\u0027] / float(info[\u0027max_row\u0027]) \u003c 0.5 and"},{"line_number":530,"context_line":"                    info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff and"},{"line_number":531,"context_line":"                    not diffs):"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_8e322ba7","line":528,"updated":"2018-03-05 09:53:52.000000000","message":"alternative: (see comments below) make decision here...\n\nfor example, if remote epoch \u003e this file\u0027s epoch AND number of diffs is \u003e diffs * per_diffs (i.e. more than one usync would take care of) then update_metadata and abort sync. \n\nMaybe we should always update metadata?","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1d026b6dd399098f08f7c32c4881abf4867214d2","unresolved":false,"context_lines":[{"line_number":525,"context_line":"            # more than 50% and the difference is greater than per_diff,"},{"line_number":526,"context_line":"            # rsync then do a remote merge."},{"line_number":527,"context_line":"            # NOTE: difference \u003e per_diff stops us from dropping to rsync"},{"line_number":528,"context_line":"            # on smaller containers, who have only a few rows to sync."},{"line_number":529,"context_line":"            if (rinfo[\u0027max_row\u0027] / float(info[\u0027max_row\u0027]) \u003c 0.5 and"},{"line_number":530,"context_line":"                    info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff and"},{"line_number":531,"context_line":"                    not diffs):"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_153b83d1","line":528,"in_reply_to":"ff6b8bd7_8e322ba7","updated":"2018-03-05 17:21:47.000000000","message":"Yeah, might be better. The big advantage I see in having the receiver 409 is in the upgrade path -- if a pre-sharding version of the container server goes to replicate with a sharded container, it won\u0027t know to make a different decision and will still send us rows, whereas the 409 will just be treated as an error until it upgrades.\n\nMaybe it\u0027s better for us to eat the rows in that case, though; we have no idea when it\u0027ll upgrade, which means we have no idea when it\u0027ll have a chance to get its rows to full durability.","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4e16111470792c859e31e36445ffe63cd02aa91c","unresolved":false,"context_lines":[{"line_number":531,"context_line":"        return False, diffs"},{"line_number":532,"context_line":""},{"line_number":533,"context_line":"    def _choose_replication_mode(self, node, rinfo, info, local_sync, broker,"},{"line_number":534,"context_line":"                                 http, different_region, diffs):"},{"line_number":535,"context_line":"        # if the difference in rowids between the two differs by"},{"line_number":536,"context_line":"        # more than 50% and the difference is greater than per_diff,"},{"line_number":537,"context_line":"        # rsync then do a remote merge."}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_b40b0628","line":534,"updated":"2018-03-14 22:42:40.000000000","message":"this isn\u0027t really a private implementation detail anymore is it?  python doesn\u0027t have anything like Java\u0027s \"protected\" concept and our code base doesn\u0027t much in the way of consistent patterns - but if a docstring would be a big improvement.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4e16111470792c859e31e36445ffe63cd02aa91c","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            return self._rsync_db(broker, node, http, info[\u0027id\u0027],"},{"line_number":546,"context_line":"                                  replicate_method\u003d\u0027rsync_then_merge\u0027,"},{"line_number":547,"context_line":"                                  replicate_timeout\u003d(info[\u0027count\u0027] / 2000),"},{"line_number":548,"context_line":"                                  different_region\u003ddifferent_region), diffs"},{"line_number":549,"context_line":"        # else send diffs over to the remote server"},{"line_number":550,"context_line":"        return self._usync_db(max(rinfo[\u0027point\u0027], local_sync),"},{"line_number":551,"context_line":"                              broker, http, rinfo[\u0027id\u0027], info[\u0027id\u0027],"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_373db88e","line":548,"updated":"2018-03-14 22:42:40.000000000","message":"oh MAN, it SO easy to miss this return value is a tuple!!!","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a21c15258d45766f10de545b56c06dab2cfeb68d","unresolved":false,"context_lines":[{"line_number":492,"context_line":"        \"\"\""},{"line_number":493,"context_line":"        diffs \u003d 0"},{"line_number":494,"context_line":"        success \u003d True"},{"line_number":495,"context_line":"        # for sub_broker in broker.get_brokers()[:-1]:"},{"line_number":496,"context_line":"        #     sub_success, diffs \u003d self._repl_db_to_node("},{"line_number":497,"context_line":"        #         node, sub_broker, partition, sub_broker.get_replication_info(),"},{"line_number":498,"context_line":"        #         different_region\u003ddifferent_region, diffs\u003ddiffs)"}],"source_content_type":"text/x-python","patch_set":10,"id":"9f6a8fd7_b37abd21","line":495,"range":{"start_line":495,"start_character":49,"end_line":495,"end_character":52},"updated":"2018-04-17 21:51:00.000000000","message":"Good -- this bit always made me nervous. I mean, what if we had some case where broker.get_brokers()[-1] !\u003d broker?","commit_id":"628dbd3220fc12f855df5bef634bc334a9f11c05"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a21c15258d45766f10de545b56c06dab2cfeb68d","unresolved":false,"context_lines":[{"line_number":517,"context_line":"        #     # broker, just like when we first created the shard broker."},{"line_number":518,"context_line":"        sub_success, _ \u003d self._repl_db_to_node("},{"line_number":519,"context_line":"            node, broker, partition, info, different_region\u003ddifferent_region,"},{"line_number":520,"context_line":"            diffs\u003ddiffs)"},{"line_number":521,"context_line":"        return success \u0026 sub_success"},{"line_number":522,"context_line":""},{"line_number":523,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"}],"source_content_type":"text/x-python","patch_set":10,"id":"9f6a8fd7_f39d9559","line":520,"range":{"start_line":520,"start_character":12,"end_line":520,"end_character":23},"updated":"2018-04-17 21:51:00.000000000","message":"!!\n\nDoes this mean we can get rid of diffs again!? Yay! (I think)","commit_id":"628dbd3220fc12f855df5bef634bc334a9f11c05"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":872,"context_line":"        drive, partition, hsh \u003d replicate_args"},{"line_number":873,"context_line":"        if not check_drive(self.root, drive, self.mount_check):"},{"line_number":874,"context_line":"            return Response(status\u003d\u0027507 %s is not mounted\u0027 % drive)"},{"line_number":875,"context_line":""},{"line_number":876,"context_line":"        db_file \u003d os.path.join(self.root, drive,"},{"line_number":877,"context_line":"                               storage_directory(self.datadir, partition, hsh),"},{"line_number":878,"context_line":"                               hsh + \u0027.db\u0027)"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_85e07cce","side":"PARENT","line":875,"updated":"2018-04-18 15:10:17.000000000","message":"no empty line on master!","commit_id":"f1138dc0fe915f2f2466c997c44074189837ff7a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":536,"context_line":"    def cleanup_post_replicate(self, broker, shouldbehere, responses):"},{"line_number":537,"context_line":"        \"\"\""},{"line_number":538,"context_line":"        Cleanup any database state if needed."},{"line_number":539,"context_line":""},{"line_number":540,"context_line":"        :return success: bool, False indicates an error occurred"},{"line_number":541,"context_line":"        \"\"\""},{"line_number":542,"context_line":"        if not shouldbehere:"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_a5c5803e","line":539,"updated":"2018-04-18 15:10:17.000000000","message":"nit: params docstring needed","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":536,"context_line":"    def cleanup_post_replicate(self, broker, shouldbehere, responses):"},{"line_number":537,"context_line":"        \"\"\""},{"line_number":538,"context_line":"        Cleanup any database state if needed."},{"line_number":539,"context_line":""},{"line_number":540,"context_line":"        :return success: bool, False indicates an error occurred"},{"line_number":541,"context_line":"        \"\"\""},{"line_number":542,"context_line":"        if not shouldbehere:"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_64ae8199","line":539,"in_reply_to":"9f6a8fd7_a5c5803e","updated":"2018-04-19 00:18:42.000000000","message":"Done","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"}],"swift/container/backend.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":405,"context_line":"            index will be appended to each key in ``info``"},{"line_number":406,"context_line":"        \"\"\""},{"line_number":407,"context_line":"        prefix \u003d \u0027X-Container-Sysmeta-Shard-\u0027"},{"line_number":408,"context_line":"        if not node:"},{"line_number":409,"context_line":"            suffix \u003d \u0027\u0027"},{"line_number":410,"context_line":"        elif node[\u0027index\u0027] \u003d\u003d \u0027handoff\u0027:"},{"line_number":411,"context_line":"            # TODO: maybe *all* nodes should use this?"},{"line_number":412,"context_line":"            suffix \u003d \u0027-%s-%s-%s\u0027 % (node[\u0027ip\u0027].replace(\u0027:\u0027, \u0027/\u0027),"},{"line_number":413,"context_line":"                                    node[\u0027port\u0027], node[\u0027device\u0027])"},{"line_number":414,"context_line":"        else:"},{"line_number":415,"context_line":"            suffix \u003d \u0027-%d\u0027 % node[\u0027index\u0027] if node else \u0027\u0027"},{"line_number":416,"context_line":"        timestamp \u003d Timestamp.now()"},{"line_number":417,"context_line":"        metadata \u003d dict("},{"line_number":418,"context_line":"            (\u0027%s%s%s\u0027 % (prefix, key, suffix),"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_eee5e70b","line":415,"range":{"start_line":408,"start_character":8,"end_line":415,"end_character":58},"updated":"2018-03-05 09:53:52.000000000","message":"we need to stop using node to track sharding porgress metadata and use db id instead. I\u0027ll try to get a patch for that done today.","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1d026b6dd399098f08f7c32c4881abf4867214d2","unresolved":false,"context_lines":[{"line_number":405,"context_line":"            index will be appended to each key in ``info``"},{"line_number":406,"context_line":"        \"\"\""},{"line_number":407,"context_line":"        prefix \u003d \u0027X-Container-Sysmeta-Shard-\u0027"},{"line_number":408,"context_line":"        if not node:"},{"line_number":409,"context_line":"            suffix \u003d \u0027\u0027"},{"line_number":410,"context_line":"        elif node[\u0027index\u0027] \u003d\u003d \u0027handoff\u0027:"},{"line_number":411,"context_line":"            # TODO: maybe *all* nodes should use this?"},{"line_number":412,"context_line":"            suffix \u003d \u0027-%s-%s-%s\u0027 % (node[\u0027ip\u0027].replace(\u0027:\u0027, \u0027/\u0027),"},{"line_number":413,"context_line":"                                    node[\u0027port\u0027], node[\u0027device\u0027])"},{"line_number":414,"context_line":"        else:"},{"line_number":415,"context_line":"            suffix \u003d \u0027-%d\u0027 % node[\u0027index\u0027] if node else \u0027\u0027"},{"line_number":416,"context_line":"        timestamp \u003d Timestamp.now()"},{"line_number":417,"context_line":"        metadata \u003d dict("},{"line_number":418,"context_line":"            (\u0027%s%s%s\u0027 % (prefix, key, suffix),"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_d5342be1","line":415,"range":{"start_line":408,"start_character":8,"end_line":415,"end_character":58},"in_reply_to":"ff6b8bd7_eee5e70b","updated":"2018-03-05 17:21:47.000000000","message":"Yeah, that sounds lovely :-)","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":374,"context_line":"    def requires_sharding(self):"},{"line_number":375,"context_line":"        db_state \u003d self.get_db_state()"},{"line_number":376,"context_line":"        return (db_state \u003d\u003d SHARDING or"},{"line_number":377,"context_line":"                (db_state \u003d\u003d UNSHARDED and self.get_shard_ranges()))"},{"line_number":378,"context_line":""},{"line_number":379,"context_line":"    def reload_db_files(self):"},{"line_number":380,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_e4729149","line":377,"updated":"2018-04-19 00:18:42.000000000","message":"+1","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"}],"swift/container/replicator.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":340,"context_line":"            broker, remote_info)"},{"line_number":341,"context_line":"        db_state \u003d broker.get_db_state()"},{"line_number":342,"context_line":"        if db_state \u003d\u003d SHARDED and remote_info.get(\u0027db_state\u0027) !\u003d db_state:"},{"line_number":343,"context_line":"            # If this db is sharded and the other end hasn\u0027t finished doing so"},{"line_number":344,"context_line":"            # refuse to replicate and tell them where to ship rows instead"},{"line_number":345,"context_line":"            # TODO: maybe expand this state to include SHARDING and just send"},{"line_number":346,"context_line":"            # whatever ranges we\u0027ve identified so far"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_035cb449","line":343,"range":{"start_line":343,"start_character":22,"end_line":343,"end_character":78},"updated":"2018-03-05 09:53:52.000000000","message":"maybe a better condition is \u0027if remote source file epoch is older than local epoch\u0027\n\nI think we agreed that when local and node are sharding and both have files:\n\n{hash_epoch1.db, hash.db}\n\nthen it is ok to sync row from remote hash_epoch1.db to local hash_epoch1.db, but we\u0027d like to avoid syncing lots of rows from remote hash.db to local hash_epoch1.db.\n\nSo instead of sending db_state, sending the remote source filename might be sufficient (but still hits the brittle args problem above)","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":342,"context_line":"        if db_state \u003d\u003d SHARDED and remote_info.get(\u0027db_state\u0027) !\u003d db_state:"},{"line_number":343,"context_line":"            # If this db is sharded and the other end hasn\u0027t finished doing so"},{"line_number":344,"context_line":"            # refuse to replicate and tell them where to ship rows instead"},{"line_number":345,"context_line":"            # TODO: maybe expand this state to include SHARDING and just send"},{"line_number":346,"context_line":"            # whatever ranges we\u0027ve identified so far"},{"line_number":347,"context_line":"            resp.status \u003d HTTP_CONFLICT"},{"line_number":348,"context_line":"            data \u003d json.loads(resp.body)"},{"line_number":349,"context_line":"            data[\u0027shard_ranges\u0027] \u003d ["}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_23e9b87f","line":346,"range":{"start_line":345,"start_character":20,"end_line":346,"end_character":53},"updated":"2018-03-05 09:53:52.000000000","message":"+1\n\nI\u0027m becoming convinced that we should find ranges in a single batch to make life simpler, so here we\u0027d have a complete set to return.","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd77da8304f462c0a3a77f0f34f83a40207998ed","unresolved":false,"context_lines":[{"line_number":344,"context_line":"            # refuse to replicate and tell them where to ship rows instead"},{"line_number":345,"context_line":"            # TODO: maybe expand this state to include SHARDING and just send"},{"line_number":346,"context_line":"            # whatever ranges we\u0027ve identified so far"},{"line_number":347,"context_line":"            resp.status \u003d HTTP_CONFLICT"},{"line_number":348,"context_line":"            data \u003d json.loads(resp.body)"},{"line_number":349,"context_line":"            data[\u0027shard_ranges\u0027] \u003d ["},{"line_number":350,"context_line":"                dict(sr) for sr in broker.get_shard_ranges()]"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_8e030b0e","line":347,"range":{"start_line":347,"start_character":26,"end_line":347,"end_character":39},"updated":"2018-03-05 09:53:52.000000000","message":"how about shifting the decision to the *sender*, i.e.\n\nalways return shard ranges in response body, and note that metadata already includes shard epoch (or should do). The latter is sufficient for the sender to deduce whether it is about to sync rows from an older db to a fresher db.\n\nThis gets around the sync args problem (sender does not need to send any more args) and also allows sender to make the decision based on number of rows to sync - as I remember it, the concern was to not sync lots of rows, whereas a small number of rows is acceptable and syncing them would increase durability.","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1d026b6dd399098f08f7c32c4881abf4867214d2","unresolved":false,"context_lines":[{"line_number":344,"context_line":"            # refuse to replicate and tell them where to ship rows instead"},{"line_number":345,"context_line":"            # TODO: maybe expand this state to include SHARDING and just send"},{"line_number":346,"context_line":"            # whatever ranges we\u0027ve identified so far"},{"line_number":347,"context_line":"            resp.status \u003d HTTP_CONFLICT"},{"line_number":348,"context_line":"            data \u003d json.loads(resp.body)"},{"line_number":349,"context_line":"            data[\u0027shard_ranges\u0027] \u003d ["},{"line_number":350,"context_line":"                dict(sr) for sr in broker.get_shard_ranges()]"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff6b8bd7_15ae23e7","line":347,"range":{"start_line":347,"start_character":26,"end_line":347,"end_character":39},"in_reply_to":"ff6b8bd7_8e030b0e","updated":"2018-03-05 17:21:47.000000000","message":"\u003e the concern was to not sync lots of rows, whereas a small number of rows is acceptable\n\nThat\u0027s true -- I wasn\u0027t entirely happy with the conditions I set, but wanted to get what I\u0027d come up with on the plane out there to look at. I agree, syncing a small number of rows should be fine.\n\n\u003e syncing them would increase durability\n\n... *immediately*. I still feel that having the sharder cleave rows increases durability, we just have to wait a bit longer :-)","commit_id":"7a34c3cb6a9eccf9f49e8e9bde893bff5dd3237f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":82,"context_line":""},{"line_number":83,"context_line":"    def _choose_replication_mode(self, node, rinfo, info, local_sync, broker,"},{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_0fc7d514","line":86,"range":{"start_line":85,"start_character":11,"end_line":86,"end_character":36},"updated":"2018-03-08 10:49:24.000000000","message":"I think this is the wrong condition:\n\n(1) shard_max_row is not a good indicator of remote state:\n\n- it will be \u003e 0 after a container has shrunk and all shard rows are deleted\n\n- now we persist own shard range in shard range table, we may want to have own shard range exists for any state of a container\n\n- we may evolve to have shard ranges defined but not yet be sharding (e.g. externally managed sharding)\n\n(2) if local is sharding and remote sharding then we DO want to replicate between the fresher, new epoch dbs, but not send large diffs from older to fresher - I\u0027m not sure this gets us that.\n\nI think it all hangs off relative epochs. We need to add epoch to replication info i.e. the epoch part of the db file . Then:\n\n  if (rinfo[\u0027epoch\u0027] \u003e info[\u0027epoch\u0027] and\n          info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e diff_limit):\n      # don\u0027t send object rows","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4e16111470792c859e31e36445ffe63cd02aa91c","unresolved":false,"context_lines":[{"line_number":82,"context_line":""},{"line_number":83,"context_line":"    def _choose_replication_mode(self, node, rinfo, info, local_sync, broker,"},{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_77c33058","line":86,"range":{"start_line":85,"start_character":11,"end_line":86,"end_character":36},"in_reply_to":"df7087c5_0fc7d514","updated":"2018-03-14 22:42:40.000000000","message":"the variable info is local db broker info - the value returned by get_db_state is already @ info[\u0027db_state\u0027] - we should use that.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"694cce4db745cc5a91c68703295dd03e3baffa5d","unresolved":false,"context_lines":[{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_3b07831f","line":87,"updated":"2018-03-08 06:21:12.000000000","message":"whats our definition of 1 usync away?\n\nIn a normal usync we\u0027d do self.per_diff self.max_diffs times. So a single usync is self.per_diff * self.max_diffs.\nMeaning \u003e self.per_diff (above) is only a part of a diff+ away.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_4f770d3c","line":87,"in_reply_to":"df7087c5_3b07831f","updated":"2018-03-08 10:49:24.000000000","message":"+1 normally 1 usync would send up to max_diffs * per_diff which by default is 100,000 rows\n\nWe can pick any number we want as a cap - in Dublin what I was hearing was O(10,0000) or even O(100,000) would be OK but O(10,000,000) is not ok. So 1 usync worth might make sense as a cap.\n\nBut if we do sync some object rows on this cycle and then info[\u0027max_row\u0027] increases before the next cycle[1] ...will we do the same? How can we cap the number of rows the older db will ever sync to the newer db? ... make the cap zero? or do the sync points allow us to infer that we had our one chance to send rows and now we should always abort?\n\n[1] hopefully the local moved to shardnig and so the older local db became read only, but maybe not, or it had some more object rows merged to it before then.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"75d9ffcb5f610867076f64949d4671508958a123","unresolved":false,"context_lines":[{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_e9cdb35d","line":87,"in_reply_to":"df7087c5_4f770d3c","updated":"2018-03-08 19:51:09.000000000","message":"Take it up with db_replicator -- I just want to avoid an rsync_then_merge; see https://github.com/openstack/swift/blob/feature/deep/swift/common/db_replicator.py#L532\n\nI could update the commit message if it\u0027d make you feel any better :P","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9750aa490bf41e60d12926039acda4f41ddf4067","unresolved":false,"context_lines":[{"line_number":84,"context_line":"                                 http, different_region, diffs):"},{"line_number":85,"context_line":"        if rinfo.get(\u0027shard_max_row\u0027, -1) \u003e 0 and broker.get_db_state() in ("},{"line_number":86,"context_line":"                UNSHARDED, SHARDING) and ("},{"line_number":87,"context_line":"                info[\u0027max_row\u0027] - rinfo[\u0027max_row\u0027] \u003e self.per_diff):"},{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_aaf0fb68","line":87,"in_reply_to":"df7087c5_e9cdb35d","updated":"2018-03-09 10:20:11.000000000","message":"Sure this avoids an rsync_then_merge but it also avoids what could have been a usync of \u003e1000 rows. If the goal is just to avoid rsync_then_merge then we should have a helper use_usync() to call here and at [1] and apply the same condition in each place.\n\nBut if the goal is to avoid shipping \"too many\" rows to the remote then we need to consider (a) what is the cap on number of rows to usync? and (b) the possibility of repeated usyncs if local max row increases. \n\nApproach (1) is to have quite a small cap e.g. 1000 as it is here, allowing for the (unlikely?) case that we might send another 1000 if local max row increases.\n\nA different approach (2) would be to have a larger cap e.g. 100, 000 and make sure we only send rows once. We might be able to enforce \u0027once\u0027 by aborting if the remote id is in outgoing syncs? So the behaviour would become:\n - never rsync_then_merge\n - allow up to cap rows to usync first time we sync to remote\n - abort if we sync\u0027d to this remote before \n\n[1] https://github.com/openstack/swift/blob/feature/deep/swift/common/db_replicator.py#L532","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"},{"line_number":91,"context_line":"            self._fetch_and_merge_shard_ranges(http, broker)"},{"line_number":92,"context_line":"            # ... but don\u0027t touch objects"},{"line_number":93,"context_line":"            self.logger.warning("},{"line_number":94,"context_line":"                \u0027%s is more than one usync away from sharding/sharded peer \u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_af2109b7","line":91,"range":{"start_line":91,"start_character":12,"end_line":91,"end_character":60},"updated":"2018-03-08 10:49:24.000000000","message":"can we not add shard ranges to the replication info and always merge to local, just like we now do with metadata?\n\nWhatever, should we not *always* get and merge shard ranges? The, even if object rows are sync\u0027d on this cycle, the local can get on with cleaving, and not wait for the next cycle?","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9750aa490bf41e60d12926039acda4f41ddf4067","unresolved":false,"context_lines":[{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"},{"line_number":91,"context_line":"            self._fetch_and_merge_shard_ranges(http, broker)"},{"line_number":92,"context_line":"            # ... but don\u0027t touch objects"},{"line_number":93,"context_line":"            self.logger.warning("},{"line_number":94,"context_line":"                \u0027%s is more than one usync away from sharding/sharded peer \u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_2a6aabb2","line":91,"range":{"start_line":91,"start_character":12,"end_line":91,"end_character":60},"in_reply_to":"df7087c5_1def0185","updated":"2018-03-09 10:20:11.000000000","message":"we *send* shard ranges in usync but don\u0027t fetch them, so if the row diff is small enough to do a usync then this local has to wait for next cycle (either inbound or outbound replication) before it gets shard ranges","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"75d9ffcb5f610867076f64949d4671508958a123","unresolved":false,"context_lines":[{"line_number":88,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":89,"context_line":"            self._sync_other_items(broker, http, info[\u0027id\u0027])"},{"line_number":90,"context_line":"            # Importantly, grab whatever the remote has, too"},{"line_number":91,"context_line":"            self._fetch_and_merge_shard_ranges(http, broker)"},{"line_number":92,"context_line":"            # ... but don\u0027t touch objects"},{"line_number":93,"context_line":"            self.logger.warning("},{"line_number":94,"context_line":"                \u0027%s is more than one usync away from sharding/sharded peer \u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_1def0185","line":91,"range":{"start_line":91,"start_character":12,"end_line":91,"end_character":60},"in_reply_to":"df7087c5_af2109b7","updated":"2018-03-08 19:51:09.000000000","message":"\u003e can we not add shard ranges to the replication info\n\nWe could -- we but there are a couple downsides:\n\n- sync responses grow bigger\n- if/when we have dbs that require a significant number of shards (\u003e10,000?), I\u0027m fairly certain we\u0027ll want to sync them based on row_id similar to what we do for objects -- and _fetch_and_merge_shard_ranges seems like a sane hook to do that.\n\n\u003e should we not *always* get and merge shard ranges?\n\nWe will -- either we fetch and merge here, or we usync in super. Or is the concern that _in_sync could come back True?","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":105,"context_line":"            different_region, diffs)"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"    def _fetch_and_merge_shard_ranges(self, http, broker):"},{"line_number":108,"context_line":"        resp \u003d http.replicate(\u0027get_shard_ranges\u0027)"},{"line_number":109,"context_line":"        broker.merge_shard_ranges(json.loads(resp.data))"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    def _initialize_broker(self, device, part, account, container, **kwargs):"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_5ee6e018","line":108,"range":{"start_line":108,"start_character":30,"end_line":108,"end_character":49},"updated":"2018-03-08 10:49:24.000000000","message":"what happens if remote has not been upgraded? The getattr(op) in dispatch doesn\u0027t look very forgiving :(\n\nah, but we wouldn\u0027t get shard_max_row if remote wasn\u0027t upgraded?\n\nI still wonder about dropping the shard ranges into replication info though.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"75d9ffcb5f610867076f64949d4671508958a123","unresolved":false,"context_lines":[{"line_number":105,"context_line":"            different_region, diffs)"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"    def _fetch_and_merge_shard_ranges(self, http, broker):"},{"line_number":108,"context_line":"        resp \u003d http.replicate(\u0027get_shard_ranges\u0027)"},{"line_number":109,"context_line":"        broker.merge_shard_ranges(json.loads(resp.data))"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    def _initialize_broker(self, device, part, account, container, **kwargs):"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_fd9ecd05","line":108,"range":{"start_line":108,"start_character":30,"end_line":108,"end_character":49},"in_reply_to":"df7087c5_5ee6e018","updated":"2018-03-08 19:51:09.000000000","message":"\u003e The getattr(op) in dispatch doesn\u0027t look very forgiving :(\n\nIt is not -- you get 500s and tracebacks. I played around with it a little yesterday to see if I could do a double-dispatch or a direct call to some other function you\u0027re not supposed to be calling, but that\u0027s neither here nor there :-)\n\n\u003e we wouldn\u0027t get shard_max_row if remote wasn\u0027t upgraded\n\nBingo.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"694cce4db745cc5a91c68703295dd03e3baffa5d","unresolved":false,"context_lines":[{"line_number":341,"context_line":"        return info"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":344,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":345,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":346,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_5b56f72b","line":344,"updated":"2018-03-08 06:21:12.000000000","message":"I\u0027m still jetlagged so haven\u0027t throught it through, but should we sync all the shard_ranges? Ie. including the brokers own?\n\n  broker.get_shard_ranges(include_deleted\u003dTrue, include_own\u003dTrue)","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":341,"context_line":"        return info"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":344,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":345,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":346,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_fe3b3499","line":344,"in_reply_to":"df7087c5_5b56f72b","updated":"2018-03-08 10:49:24.000000000","message":"+1\n\nin general we never want a container to have shard ranges but not have its own sharding context (i.e. own shard range plus bits of metadata like Root) so we must ship own shard range here.\n\naside: please shout if you spot other cases where a contaienr might end up with only partial sharding info - e.g. a failure condition - we should ensure the outcome is always benign.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"75d9ffcb5f610867076f64949d4671508958a123","unresolved":false,"context_lines":[{"line_number":341,"context_line":"        return info"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":344,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":345,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":346,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_7d92bdf8","line":344,"in_reply_to":"df7087c5_fe3b3499","updated":"2018-03-08 19:51:09.000000000","message":"Yeah, almost certainly. Will do.","commit_id":"fae52c50792f48716beeecb2b5cbb00a5a32b7b0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"72a88e482d7070fdecb6b35db6b01f1b5a3436af","unresolved":false,"context_lines":[{"line_number":83,"context_line":"        return super(ContainerReplicator, self)._handle_sync_response("},{"line_number":84,"context_line":"            node, response, info, broker, http, different_region, diffs\u003ddiffs)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"    def _rsync_db(self, broker, device, http, local_id,"},{"line_number":87,"context_line":"                 replicate_method\u003d\u0027complete_rsync\u0027, replicate_timeout\u003dNone,"},{"line_number":88,"context_line":"                 different_region\u003dFalse):"},{"line_number":89,"context_line":"        shard_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":7,"id":"bf659307_775171d8","line":86,"updated":"2018-04-16 18:34:26.000000000","message":"We most likely still want https://review.openstack.org/550696 (or something like it) so we don\u0027t push 100s of 1000s of rows through our tiny DBs that have already sharded.","commit_id":"33d349116afe39962de9ec07f20d167563aad04b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5bc3272919e3c3e072cc0f0de25a604b4567c036","unresolved":false,"context_lines":[{"line_number":345,"context_line":"        return info"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":348,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":349,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":350,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_71c1812e","line":348,"range":{"start_line":348,"start_character":47,"end_line":348,"end_character":67},"updated":"2018-03-20 15:31:35.000000000","message":"also include_own","commit_id":"33d349116afe39962de9ec07f20d167563aad04b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"72a88e482d7070fdecb6b35db6b01f1b5a3436af","unresolved":false,"context_lines":[{"line_number":345,"context_line":"        return info"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def get_shard_ranges(self, broker, args):"},{"line_number":348,"context_line":"        shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":349,"context_line":"        return HTTPOk(headers\u003d{\u0027Content-Type\u0027: \u0027application/json\u0027},"},{"line_number":350,"context_line":"                      body\u003djson.dumps([dict(sr) for sr in shard_ranges]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"bf659307_f7458118","line":348,"range":{"start_line":348,"start_character":47,"end_line":348,"end_character":67},"in_reply_to":"df7087c5_71c1812e","updated":"2018-04-16 18:34:26.000000000","message":"In particular, want to make sure that own_shard_range is in an appropriate state such that we *will* cleave when the sharder runs.","commit_id":"33d349116afe39962de9ec07f20d167563aad04b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a21c15258d45766f10de545b56c06dab2cfeb68d","unresolved":false,"context_lines":[{"line_number":91,"context_line":"    def _choose_replication_mode(self, node, rinfo, info, local_sync, broker,"},{"line_number":92,"context_line":"                                 http, different_region, diffs):"},{"line_number":93,"context_line":"        shard_ranges \u003d broker.get_shard_ranges()"},{"line_number":94,"context_line":"        # TODO: also check that own shard range state \u003e ACTIVE?"},{"line_number":95,"context_line":"        if shard_ranges:"},{"line_number":96,"context_line":"            # Keep replicating shard ranges -- may have newer info than remote"},{"line_number":97,"context_line":"            success \u003d self._sync_other_items(broker, http, info[\u0027id\u0027])"}],"source_content_type":"text/x-python","patch_set":10,"id":"9f6a8fd7_9f9bfbc7","line":94,"updated":"2018-04-17 21:51:00.000000000","message":"Almost certainly. Otherwise, there will be... implications... with https://trello.com/c/dMvGnQsl/246-get-more-than-own-shard-range-in-audit and https://review.openstack.org/#/c/560114/3/swift/container/sharder.py@725\n\nIt\u0027s definitely not a problem *yet*, but something for us to keep in mind...","commit_id":"628dbd3220fc12f855df5bef634bc334a9f11c05"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":66,"context_line":"        return sync_args"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":69,"context_line":"                              different_region\u003dFalse, diffs\u003d0):"},{"line_number":70,"context_line":"        parent \u003d super(ContainerReplicator, self)"},{"line_number":71,"context_line":"        if is_success(response.status):"},{"line_number":72,"context_line":"            remote_info \u003d json.loads(response.data)"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_65068889","side":"PARENT","line":69,"range":{"start_line":69,"start_character":54,"end_line":69,"end_character":61},"updated":"2018-04-18 15:10:17.000000000","message":":\u0027( bye bye diffs","commit_id":"f1138dc0fe915f2f2466c997c44074189837ff7a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":102,"context_line":"                broker.db_file,"},{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_2584100c","line":105,"range":{"start_line":105,"start_character":24,"end_line":105,"end_character":31},"updated":"2018-04-18 15:10:17.000000000","message":"IDK if this name is too alarming - could be \u0027shard_ranges\u0027??","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3f2026a11b9a32cc7aca25c554bca32e096a4f35","unresolved":false,"context_lines":[{"line_number":102,"context_line":"                broker.db_file,"},{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_5d99d46b","line":105,"range":{"start_line":105,"start_character":24,"end_line":105,"end_character":31},"in_reply_to":"9f6a8fd7_2584100c","updated":"2018-04-18 18:36:13.000000000","message":"Or maybe \"deferred\"? IDK","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":102,"context_line":"                broker.db_file,"},{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_047ea518","line":105,"range":{"start_line":105,"start_character":24,"end_line":105,"end_character":31},"in_reply_to":"9f6a8fd7_5d99d46b","updated":"2018-04-19 00:18:42.000000000","message":"Done","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("},{"line_number":109,"context_line":"            node, rinfo, info, local_sync, broker, http,"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_a59ee060","line":106,"range":{"start_line":106,"start_character":19,"end_line":106,"end_character":26},"updated":"2018-04-18 15:10:17.000000000","message":"My thought is that this method has succeeded if we successfully replicated shard ranges. Then, in cleanup_post_replication, we now decide whether to delete the db or not.","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("},{"line_number":109,"context_line":"            node, rinfo, info, local_sync, broker, http,"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_844cf59e","line":106,"range":{"start_line":106,"start_character":19,"end_line":106,"end_character":26},"in_reply_to":"9f6a8fd7_7d9cd85c","updated":"2018-04-19 00:18:42.000000000","message":"this is brilliant!","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3f2026a11b9a32cc7aca25c554bca32e096a4f35","unresolved":false,"context_lines":[{"line_number":103,"context_line":"                \u0027%(ip)s:%(port)s/%(device)s\u0027 % node,"},{"line_number":104,"context_line":"                len(shard_ranges))"},{"line_number":105,"context_line":"            self.stats[\u0027aborted\u0027] +\u003d 1"},{"line_number":106,"context_line":"            return success"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"        return super(ContainerReplicator, self)._choose_replication_mode("},{"line_number":109,"context_line":"            node, rinfo, info, local_sync, broker, http,"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_7d9cd85c","line":106,"range":{"start_line":106,"start_character":19,"end_line":106,"end_character":26},"in_reply_to":"9f6a8fd7_a59ee060","updated":"2018-04-18 18:36:13.000000000","message":"Yeah, that\u0027s probably a good idea.","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"}],"test/unit/container/test_replicator.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf88e1d6fbc7713d1199212a17ed38d23f8df154","unresolved":false,"context_lines":[{"line_number":2038,"context_line":"            local_broker, 1, repl_conf\u003d{\u0027per_diff\u0027: 1})"},{"line_number":2039,"context_line":""},{"line_number":2040,"context_line":"        self.assertEqual("},{"line_number":2041,"context_line":"            [\u0027sync\u0027, \u0027rsync_then_merge\u0027, \u0027sync\u0027, \u0027merge_items\u0027, \u0027merge_items\u0027,"},{"line_number":2042,"context_line":"             \u0027merge_syncs\u0027], [call[0] for call in repl_calls])"},{"line_number":2043,"context_line":"        self.assertEqual(1, daemon.stats[\u0027remote_merge\u0027])"},{"line_number":2044,"context_line":"        self.assertEqual(1, daemon.stats[\u0027diff\u0027])"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_0ad3433a","side":"PARENT","line":2041,"range":{"start_line":2041,"start_character":64,"end_line":2041,"end_character":78},"updated":"2018-03-08 10:49:24.000000000","message":"I wasn\u0027t expecting this change - we\u0027re now refusing to sync a single row from fresh db to fresh db?","commit_id":"7796ecea02a136a661ee4990026683bd7b786e07"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fff5ecfeee7e420b8e664b53db123d69863ac987","unresolved":false,"context_lines":[{"line_number":2038,"context_line":"            local_broker, 1, repl_conf\u003d{\u0027per_diff\u0027: 1})"},{"line_number":2039,"context_line":""},{"line_number":2040,"context_line":"        self.assertEqual("},{"line_number":2041,"context_line":"            [\u0027sync\u0027, \u0027rsync_then_merge\u0027, \u0027sync\u0027, \u0027merge_items\u0027, \u0027merge_items\u0027,"},{"line_number":2042,"context_line":"             \u0027merge_syncs\u0027], [call[0] for call in repl_calls])"},{"line_number":2043,"context_line":"        self.assertEqual(1, daemon.stats[\u0027remote_merge\u0027])"},{"line_number":2044,"context_line":"        self.assertEqual(1, daemon.stats[\u0027diff\u0027])"}],"source_content_type":"text/x-python","patch_set":5,"id":"df7087c5_53cdc082","side":"PARENT","line":2041,"range":{"start_line":2041,"start_character":64,"end_line":2041,"end_character":78},"in_reply_to":"df7087c5_0ad3433a","updated":"2018-03-15 22:28:55.000000000","message":"Not if there\u0027s a large diff before we ever get to the fresh dbs! We *have* to treat aborts as failures, otherwise the handoff will delete the big DB -- and since the big db fails, we fall under the\n\n # state of the destination is uncertain, bail out\n\ncomment in db_replicator.py","commit_id":"7796ecea02a136a661ee4990026683bd7b786e07"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":1192,"context_line":"        self.assertTrue(res)"},{"line_number":1193,"context_line":"        self.assertTrue(os.path.exists(broker.db_file))"},{"line_number":1194,"context_line":""},{"line_number":1195,"context_line":"        # db sharding, nor deleted"},{"line_number":1196,"context_line":"        self._goto_sharding_state(broker, Timestamp.now())"},{"line_number":1197,"context_line":"        self.assertTrue(broker.requires_sharding())  # sanity check"},{"line_number":1198,"context_line":"        res \u003d daemon.cleanup_post_replicate(broker, False, [True] * 3)"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_84ddd515","line":1195,"updated":"2018-04-19 00:18:42.000000000","message":"s/nor/not\n\nDone","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":1214,"context_line":"        # db sharded, should not be here, replication ok, deleted"},{"line_number":1215,"context_line":"        res \u003d daemon.cleanup_post_replicate(broker, False, [True] * 3)"},{"line_number":1216,"context_line":"        self.assertTrue(res)"},{"line_number":1217,"context_line":"        self.assertFalse(os.path.exists(broker.db_file))"},{"line_number":1218,"context_line":""},{"line_number":1219,"context_line":"    def test_sync_shard_ranges(self):"},{"line_number":1220,"context_line":"        put_timestamp \u003d Timestamp.now().internal"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_249e4954","line":1217,"updated":"2018-04-19 00:18:42.000000000","message":"these assertions are great!","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"740617a461429f34168ed84502c01362ceef81b8","unresolved":false,"context_lines":[{"line_number":1530,"context_line":"                                            remote_node_index,"},{"line_number":1531,"context_line":"                                            repl_conf,"},{"line_number":1532,"context_line":"                                            expected_shard_ranges,"},{"line_number":1533,"context_line":"                                            expect_success\u003dTrue):"},{"line_number":1534,"context_line":"        # expected_shard_ranges is expected final list of sync\u0027d ranges"},{"line_number":1535,"context_line":"        daemon, repl_calls, rsync_calls \u003d self.check_replicate("},{"line_number":1536,"context_line":"            local_broker, remote_node_index, repl_conf,"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_85171ca4","line":1533,"range":{"start_line":1533,"start_character":44,"end_line":1533,"end_character":63},"updated":"2018-04-18 15:10:17.000000000","message":"My thinking is that in the normal case we expect success","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f28d531de28aec0b4bd5fa9695b40874b5f4a45d","unresolved":false,"context_lines":[{"line_number":1530,"context_line":"                                            remote_node_index,"},{"line_number":1531,"context_line":"                                            repl_conf,"},{"line_number":1532,"context_line":"                                            expected_shard_ranges,"},{"line_number":1533,"context_line":"                                            expect_success\u003dTrue):"},{"line_number":1534,"context_line":"        # expected_shard_ranges is expected final list of sync\u0027d ranges"},{"line_number":1535,"context_line":"        daemon, repl_calls, rsync_calls \u003d self.check_replicate("},{"line_number":1536,"context_line":"            local_broker, remote_node_index, repl_conf,"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f6a8fd7_7f7608a7","line":1533,"range":{"start_line":1533,"start_character":44,"end_line":1533,"end_character":63},"in_reply_to":"9f6a8fd7_85171ca4","updated":"2018-04-19 00:18:42.000000000","message":"Yas!","commit_id":"5ebda3908de4398c833c88fadad4a78d63ad79b8"}]}