)]}'
{"swift/common/db_replicator.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5d027afa0769776bbfce2f1ca7e823698380c009","unresolved":false,"context_lines":[{"line_number":937,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":938,"context_line":"            objects \u003d existing_broker.get_items_since(point, 1000)"},{"line_number":939,"context_line":"            sleep()"},{"line_number":940,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":941,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":942,"context_line":"        # limit in the future."},{"line_number":943,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa0c359_d5441a77","line":940,"updated":"2018-02-15 18:38:56.000000000","message":"why did this not happen on master? makes me nervous that there is a subtle reason to NOT do this, but it is required when we sync two dbs (hash.db and a shard db) - see commit message.","commit_id":"dbcfba0c5c3845063f91008650a08748cddf9231"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ef1325d091942433f81cc1d3b2e640a718b348ba","unresolved":false,"context_lines":[{"line_number":99,"context_line":"    \"\"\""},{"line_number":100,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":101,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":102,"context_line":"        return None"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return None"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_b0012b07","line":102,"updated":"2018-02-19 06:06:57.000000000","message":"The doc string says this list maybe empty, so this should probably be [] rather then None.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfae560b57656f11684b309ca9c5f52d02c1d6ab","unresolved":false,"context_lines":[{"line_number":99,"context_line":"    \"\"\""},{"line_number":100,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":101,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":102,"context_line":"        return None"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return None"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_0d09dd2a","line":102,"in_reply_to":"3fa0c359_b0012b07","updated":"2018-02-19 19:48:16.000000000","message":"Done","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ef1325d091942433f81cc1d3b2e640a718b348ba","unresolved":false,"context_lines":[{"line_number":102,"context_line":"        return None"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return None"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"},{"line_number":107,"context_line":"    results \u003d []"},{"line_number":108,"context_line":"    for f in files:"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_5008e726","line":105,"updated":"2018-02-19 06:06:57.000000000","message":"Same here.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfae560b57656f11684b309ca9c5f52d02c1d6ab","unresolved":false,"context_lines":[{"line_number":102,"context_line":"        return None"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return None"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"},{"line_number":107,"context_line":"    results \u003d []"},{"line_number":108,"context_line":"    for f in files:"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_cd121559","line":105,"in_reply_to":"3fa0c359_5008e726","updated":"2018-02-19 19:48:16.000000000","message":"Done","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ef1325d091942433f81cc1d3b2e640a718b348ba","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        if success:"},{"line_number":512,"context_line":"            success \u003d self._repl_db_to_node(node, broker, partition, info,"},{"line_number":513,"context_line":"                                            different_region\u003ddifferent_region)"},{"line_number":514,"context_line":"        return success"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":517,"context_line":"                              different_region\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_700ba323","line":514,"updated":"2018-02-19 06:06:57.000000000","message":"OK, so in SHARDED or NOT_SHARDED we\u0027ll only send the only broker. If it\u0027s SHARDING state, then we first send the h.db then the s.db.\n\nNow, if we first send the h.db and it\u0027s so far behind the remote that is can\u0027t send all updates in a single usync.. it\u0027ll do as much as it can and update it\u0027s outgoing* and the remote will update it\u0027s incoming syncs. Then the s.db is replicated. It\u0027s coming from the same primary node, so uses the same incoming_sync on the remote side. So will ask for more. What do we do in this situation? If it\u0027s usync (which it would be), and is a s.db it would check to see if the sync row is down in the h.db and continue from there, in essence this would 2xUsncs for the price of one. However, you\u0027ve added the include_sharding\u003dFalse to broker.get_items_since, so does this mean this wont happen.\nIf not, what then happens if the h.db usyncs but not up to the end, then s.db usyncs? it\u0027ll get_items_since itself, then update the incoming sync on the remote side with the new row from s.db meaning on the next round we skip a chunk of rows.\n\nMaybe this will be OK, as these rows should get sharded out, and therefore exist so long as this wasn\u0027t the only copy and it get\u0027s corrupted.\n\n* - Note: We\u0027ve been trying hard to keep the h.db readonly (so try to avoid writing to it). This is why s.db has all the updated incoming and outgoing sync points, it\u0027s where we should _always_ write stuff. Maybe on usync + SHARDING we need to only use the s.broker (as it knows how to talk to the h.db as well as itself).\n\nOr is all thing something we need to rethink?","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ed5366bb0bca5091c4360e32efb7404ff846e482","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        if success:"},{"line_number":512,"context_line":"            success \u003d self._repl_db_to_node(node, broker, partition, info,"},{"line_number":513,"context_line":"                                            different_region\u003ddifferent_region)"},{"line_number":514,"context_line":"        return success"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":517,"context_line":"                              different_region\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_7ed03bf8","line":514,"in_reply_to":"3fa0c359_0d5a3df2","updated":"2018-02-20 08:43:36.000000000","message":"So maybe we need to add a broker.get_id(). As we move to sharding state the s.db merges the h.db\u0027s sync tables. So all we need to know if what h.db\u0027s id is (by asking).\n\nBecause we\u0027d have the sync tables and h.db\u0027s id,  while in sharding state we\u0027ll know if the sync point is \u003c h.dbs MAX_ROW. So surely we can just respond to a replication_info with the correct value and send h.dbs id until it\u0027s up to date.\n\nIf the h.db is so out of sync that it takes more then 1 usync. then that\u0027s ok because we can still store the updates in s.db using h.db\u0027s id (keeping h.db readonly).\n\nIf h.db is \u003c 1 usync behind s.db maybe we limit this replication usync to h.dbs maxrow. and start afresh with a new ID at the beginning of the s.db on the next replication pass.. so things don\u0027t get confused.\n\nThis way s.db is smart enough to sync our rows. We don\u0027t have to do 2 usyncs when sharding, we don\u0027t write to a \"read only\" db. We don\u0027t have 2 databases with the same ID.. though one does lie about who it is.. which may be considered dodgy.\n\nJust a thought.. brain storming here, maybe we should move this discussion to the etherpad ;)","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfae560b57656f11684b309ca9c5f52d02c1d6ab","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        if success:"},{"line_number":512,"context_line":"            success \u003d self._repl_db_to_node(node, broker, partition, info,"},{"line_number":513,"context_line":"                                            different_region\u003ddifferent_region)"},{"line_number":514,"context_line":"        return success"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":517,"context_line":"                              different_region\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_0d5a3df2","line":514,"in_reply_to":"3fa0c359_700ba323","updated":"2018-02-19 19:48:16.000000000","message":"@Matt thank you so much for this comment!\n\n1. yes, this version has a bug in that if the source is sharding and old h.db uses usync, but not all rows are sync\u0027d, the s.db might usync some newer rows and create a gap :/ Now I understand the idea behind the change to get_items_since() i.e. always ensure continuity of usync\u0027d rows across source dbs. #willfix\n\n2. However, while fixing this I noticed that in same scenario the destination might not have an incoming sync for the source s.db [1]. So when all syncs are deferred to the s.db usync method, its sync point is -1 so it call get_items_since(-1) and syncs the entire older h.db :( !!\n\n[1] When the shard.db is created it gets a new id w.r.t. the hash.db so appears to have never sync\u0027d with any remote db. I *think* this is probably appropriate - I\u0027d be nervous about two dbs having the same id.\n\nSo...I ended up getting each source db to usync separately but have the shard.db respect the usync progress of hash.db to avoid gaps in the rows.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9e7f65fccaa7cc90c0a0b7b7400f370033c9f45a","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        if success:"},{"line_number":512,"context_line":"            success \u003d self._repl_db_to_node(node, broker, partition, info,"},{"line_number":513,"context_line":"                                            different_region\u003ddifferent_region)"},{"line_number":514,"context_line":"        return success"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":517,"context_line":"                              different_region\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_c18400d6","line":514,"in_reply_to":"3fa0c359_7ed03bf8","updated":"2018-02-20 21:45:51.000000000","message":"IIUC, the idea is that the source lies about its id, initially sending the h.db id, until it has sync\u0027d h.db. Then the source starts to use its s.db id, gets a -1 point back, but just syncs its s.db rows (i.e. normal usync)\n\nBut that still results in two sets of \u003csync, merge_items\u003e because we\u0027d have to send a separate sync method for each source id?\n\nSo i think it might end up with same net result?? or maybe I misunderstood.\n\nI will try to gt some of these discussion points onto the etherpad.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ef1325d091942433f81cc1d3b2e640a718b348ba","unresolved":false,"context_lines":[{"line_number":912,"context_line":"    def complete_rsync(self, drive, db_file, args):"},{"line_number":913,"context_line":"        old_filename \u003d os.path.join(self.root, drive, \u0027tmp\u0027, args[0])"},{"line_number":914,"context_line":"        if args[1:]:"},{"line_number":915,"context_line":"            db_file \u003d os.path.join(os.path.dirname(db_file), args[1])"},{"line_number":916,"context_line":"        if os.path.exists(db_file):"},{"line_number":917,"context_line":"            return HTTPNotFound()"},{"line_number":918,"context_line":"        if not os.path.exists(old_filename):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_10125f59","line":915,"updated":"2018-02-19 06:06:57.000000000","message":"Remembering back to the call. What was the plan about SHARDED \u003e No file remote? I.e SHARDED handoff? Was it:\n\n    LOCAL    REMOTE\n  1. s.db     s.db\n  2. s.db     h.db\n\nThis is naming the remote to s.db in this case... So great, but I thought we\u0027d always use h.db, and let the sharder deal.. or was that just with SHARDING? I\u0027m ok the 1 in the SHARDED handoff case, in which case great, but can\u0027t remember if that\u0027s what we actually decided.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfae560b57656f11684b309ca9c5f52d02c1d6ab","unresolved":false,"context_lines":[{"line_number":912,"context_line":"    def complete_rsync(self, drive, db_file, args):"},{"line_number":913,"context_line":"        old_filename \u003d os.path.join(self.root, drive, \u0027tmp\u0027, args[0])"},{"line_number":914,"context_line":"        if args[1:]:"},{"line_number":915,"context_line":"            db_file \u003d os.path.join(os.path.dirname(db_file), args[1])"},{"line_number":916,"context_line":"        if os.path.exists(db_file):"},{"line_number":917,"context_line":"            return HTTPNotFound()"},{"line_number":918,"context_line":"        if not os.path.exists(old_filename):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_bfc6cc48","line":915,"in_reply_to":"3fa0c359_10125f59","updated":"2018-02-19 19:48:16.000000000","message":"in sharded state with remote missing, the remote gets a shard db\n\nthe \u0027rules\u0027, if you like, are:\n- always sync each source db in order of age\n- first time remote is found to be missing, rsync to creta remote db\n- always sync to the newest existing remote db\n\nso following those rules when source is sharded and remote missing:\n\n- first source to sync is shard db, so it rsyncs to create the remote\n\nwhereas is source is sharding and remote is missing, first to sync is hash.db so it rsyncs and creates remote h.db; next to sync is shard.db which syncs to existing remote h.db\n\nAlternatives may also work, but this is what we have and it seems to work!","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ef1325d091942433f81cc1d3b2e640a718b348ba","unresolved":false,"context_lines":[{"line_number":937,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":938,"context_line":"            objects \u003d existing_broker.get_items_since(point, 1000)"},{"line_number":939,"context_line":"            sleep()"},{"line_number":940,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":941,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":942,"context_line":"        # limit in the future."},{"line_number":943,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_300d1b35","line":940,"updated":"2018-02-19 06:06:57.000000000","message":"So this is syncing the imcoming syncs from the","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d0a505b71dac6cd407e13b73f4693590abe39b5a","unresolved":false,"context_lines":[{"line_number":937,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":938,"context_line":"            objects \u003d existing_broker.get_items_since(point, 1000)"},{"line_number":939,"context_line":"            sleep()"},{"line_number":940,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":941,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":942,"context_line":"        # limit in the future."},{"line_number":943,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_30719bdf","line":940,"in_reply_to":"3fa0c359_300d1b35","updated":"2018-02-19 08:17:07.000000000","message":"hmm, I never finished this one... opps... now what was I saying.\n\nI think I was going to say something about we might need this the other way around, but then double checked and it of course merges them (max of the two), so should all be correct, in fact adding this line is good cause otherwise there might have been extra useless work by the container replicators elsewhere.","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfae560b57656f11684b309ca9c5f52d02c1d6ab","unresolved":false,"context_lines":[{"line_number":937,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":938,"context_line":"            objects \u003d existing_broker.get_items_since(point, 1000)"},{"line_number":939,"context_line":"            sleep()"},{"line_number":940,"context_line":"        new_broker.merge_syncs(existing_broker.get_syncs())"},{"line_number":941,"context_line":"        # Note the following hook will need to change to using a pointer and"},{"line_number":942,"context_line":"        # limit in the future."},{"line_number":943,"context_line":"        other_items \u003d existing_broker.get_other_replication_items()"}],"source_content_type":"text/x-python","patch_set":3,"id":"3fa0c359_9fd0b0f9","line":940,"in_reply_to":"3fa0c359_30719bdf","updated":"2018-02-19 19:48:16.000000000","message":"see my comment on patchset 1, seems like this could be on master, so I\u0027m nervous why it isn\u0027t!","commit_id":"3631142137a1d85460590231ecdfaebf8038b615"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":84,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":85,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":86,"context_line":"    hash_ \u003d parts.pop(0)"},{"line_number":87,"context_line":"    other \u003d parts[0] if parts else None"},{"line_number":88,"context_line":"    return hash_, other, ext"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_f73d1bbb","line":87,"updated":"2018-02-22 01:30:55.000000000","message":"So other isn\u0027t allowed to have underscores? I wonder if something like\n\n file_hash, sep, other \u003d name.partition(\u0027_\u0027)\n if not sep:\n     other \u003d None\n\n return file_hash, other, ext\n\nmight be more obvious?","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":84,"context_line":"    name, ext \u003d os.path.splitext(filename)"},{"line_number":85,"context_line":"    parts \u003d name.split(\u0027_\u0027)"},{"line_number":86,"context_line":"    hash_ \u003d parts.pop(0)"},{"line_number":87,"context_line":"    other \u003d parts[0] if parts else None"},{"line_number":88,"context_line":"    return hash_, other, ext"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_d66576b6","line":87,"in_reply_to":"3fa0c359_f73d1bbb","updated":"2018-02-22 16:40:42.000000000","message":"In following patches \u0027other\u0027 becomes \u0027epoch\u0027 which must be a timestamp. Sorry, I realise that sometimes these incremental changes don\u0027t help.","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":100,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":101,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":102,"context_line":"        return []"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return []"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_d797bf8f","line":103,"updated":"2018-02-22 01:30:55.000000000","message":"So we\u0027ve traded a couple of stats for a listdir? I\u0027m not sure what the relative performance of those is...","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":100,"context_line":"    db_dir, db_file \u003d os.path.split(db_path)"},{"line_number":101,"context_line":"    if not os.path.exists(db_dir):"},{"line_number":102,"context_line":"        return []"},{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return []"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_36ccda68","line":103,"in_reply_to":"3fa0c359_d797bf8f","updated":"2018-02-22 16:40:42.000000000","message":"the goal was to generalise rather than rely on files being named h.db and h_shard.db, because in following patches the naming changes.","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return []"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"},{"line_number":107,"context_line":"    results \u003d []"},{"line_number":108,"context_line":"    for f in files:"},{"line_number":109,"context_line":"        hash_, other, ext \u003d parse_db_filename(f)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_17a1a7f7","line":106,"range":{"start_line":106,"start_character":4,"end_line":106,"end_character":14},"updated":"2018-02-22 01:30:55.000000000","message":"I wonder if we should actually be using the final piece of db_dir as the match_hash...","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    files \u003d os.listdir(db_dir)"},{"line_number":104,"context_line":"    if not files:"},{"line_number":105,"context_line":"        return []"},{"line_number":106,"context_line":"    match_hash, other, ext \u003d parse_db_filename(db_file)"},{"line_number":107,"context_line":"    results \u003d []"},{"line_number":108,"context_line":"    for f in files:"},{"line_number":109,"context_line":"        hash_, other, ext \u003d parse_db_filename(f)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_b606aa85","line":106,"range":{"start_line":106,"start_character":4,"end_line":106,"end_character":14},"in_reply_to":"3fa0c359_17a1a7f7","updated":"2018-02-22 16:40:42.000000000","message":"interesting. so in case a caller gives a bad db_path such as \u0027path/to/hash1/hash2.db\u0027 ?","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    for f in files:"},{"line_number":109,"context_line":"        hash_, other, ext \u003d parse_db_filename(f)"},{"line_number":110,"context_line":"        if ext !\u003d \u0027.db\u0027:"},{"line_number":111,"context_line":"            continue"},{"line_number":112,"context_line":"        if hash_ !\u003d match_hash:"},{"line_number":113,"context_line":"            continue"},{"line_number":114,"context_line":"        results.append(os.path.join(db_dir, f))"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_176c47ce","line":111,"updated":"2018-02-22 01:30:55.000000000","message":"So even if db_path exists, it may not appear in the result? Interesting...","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    for f in files:"},{"line_number":109,"context_line":"        hash_, other, ext \u003d parse_db_filename(f)"},{"line_number":110,"context_line":"        if ext !\u003d \u0027.db\u0027:"},{"line_number":111,"context_line":"            continue"},{"line_number":112,"context_line":"        if hash_ !\u003d match_hash:"},{"line_number":113,"context_line":"            continue"},{"line_number":114,"context_line":"        results.append(os.path.join(db_dir, f))"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_563c062e","line":111,"in_reply_to":"3fa0c359_176c47ce","updated":"2018-02-22 16:40:42.000000000","message":"perhaps parse_db_filename should raise an error if ext !\u003d \u0027.db\u0027","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":156,"context_line":"                    # look for any alternate db filenames"},{"line_number":157,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":158,"context_line":"                    if db_files:"},{"line_number":159,"context_line":"                        yield (partition, db_files[-1], node_id)"},{"line_number":160,"context_line":"                        continue"},{"line_number":161,"context_line":"                    try:"},{"line_number":162,"context_line":"                        os.rmdir(hash_dir)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_77eeeb22","line":159,"updated":"2018-02-22 01:30:55.000000000","message":"Just the last one?","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":156,"context_line":"                    # look for any alternate db filenames"},{"line_number":157,"context_line":"                    db_files \u003d get_db_files(object_file)"},{"line_number":158,"context_line":"                    if db_files:"},{"line_number":159,"context_line":"                        yield (partition, db_files[-1], node_id)"},{"line_number":160,"context_line":"                        continue"},{"line_number":161,"context_line":"                    try:"},{"line_number":162,"context_line":"                        os.rmdir(hash_dir)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_b6c78a12","line":159,"in_reply_to":"3fa0c359_77eeeb22","updated":"2018-02-22 16:40:42.000000000","message":"yes. the behaviour hasn\u0027t changed, it has just been generalised to not assume fixed db filenames. (note the elif in original)\n\ncurrently this would yield the h_shard.db, in future it will yield the db file with most recent epoch.","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":487,"context_line":"        with Timeout(self.node_timeout):"},{"line_number":488,"context_line":"            response \u003d http.replicate(\u0027sync\u0027, *sync_args)"},{"line_number":489,"context_line":"        if not response:"},{"line_number":490,"context_line":"            return False, 0"},{"line_number":491,"context_line":"        return self._handle_sync_response(node, response, info, broker, http,"},{"line_number":492,"context_line":"                                          different_region\u003ddifferent_region,"},{"line_number":493,"context_line":"                                          diffs\u003ddiffs)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_f7c2fb84","line":490,"range":{"start_line":490,"start_character":26,"end_line":490,"end_character":27},"updated":"2018-02-22 01:30:55.000000000","message":"Not diffs? Ah, because we use 0 to flag that we should bail early below...","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":536,"context_line":"        return success \u0026 sub_success"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":539,"context_line":"                              different_region\u003dFalse, diffs\u003d0):"},{"line_number":540,"context_line":"        if response.status \u003d\u003d HTTP_NOT_FOUND:  # completely missing, rsync"},{"line_number":541,"context_line":"            self.stats[\u0027rsync\u0027] +\u003d 1"},{"line_number":542,"context_line":"            self.logger.increment(\u0027rsyncs\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_b71473ea","line":539,"updated":"2018-02-22 01:30:55.000000000","message":"So we have to plumb this through all of\n\n- _usync_db\n- _repl_db_to_node\n- _handle_sync_response\n\nand kind of\n\n- _repl_to_node\n\n? It feels a little brittle -- could we at least not default this back to zero every step of the way? Or we could add a ReplicationContext or something that\u0027s just a bag of node/broker/partition/etc. that we pass around...","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":536,"context_line":"        return success \u0026 sub_success"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def _handle_sync_response(self, node, response, info, broker, http,"},{"line_number":539,"context_line":"                              different_region\u003dFalse, diffs\u003d0):"},{"line_number":540,"context_line":"        if response.status \u003d\u003d HTTP_NOT_FOUND:  # completely missing, rsync"},{"line_number":541,"context_line":"            self.stats[\u0027rsync\u0027] +\u003d 1"},{"line_number":542,"context_line":"            self.logger.increment(\u0027rsyncs\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_76b4a282","line":539,"in_reply_to":"3fa0c359_b71473ea","updated":"2018-02-22 16:40:42.000000000","message":"not sure what you mean by defaulting back to zero\n\nI think encapsulating diffs in a context might be ok. I\u0027d be wary of creating too much churn w.r.t. master, so maybe just a UsyncContext that hides the detail of \u0027diffs\u0027 a little?","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"}],"swift/container/replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"68c2e020c10e14bef31556e66f102ff6b730ce20","unresolved":false,"context_lines":[{"line_number":281,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"},{"line_number":282,"context_line":""},{"line_number":283,"context_line":"    def _db_file_exists(self, db_path):"},{"line_number":284,"context_line":"        return db_replicator.get_db_files(db_path)"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"    def _parse_sync_args(self, args):"},{"line_number":287,"context_line":"        parent \u003d super(ContainerReplicatorRpc, self)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_f77afb21","line":284,"updated":"2018-02-22 01:30:55.000000000","message":"Worth adding a bool()?","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"44241a9c832b36a3047f2cb082996af3fb6753bd","unresolved":false,"context_lines":[{"line_number":281,"context_line":"class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):"},{"line_number":282,"context_line":""},{"line_number":283,"context_line":"    def _db_file_exists(self, db_path):"},{"line_number":284,"context_line":"        return db_replicator.get_db_files(db_path)"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"    def _parse_sync_args(self, args):"},{"line_number":287,"context_line":"        parent \u003d super(ContainerReplicatorRpc, self)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_e28d497c","line":284,"in_reply_to":"3fa0c359_f77afb21","updated":"2018-02-22 16:40:42.000000000","message":"Done","commit_id":"029d7ab565097ce55f2955064b368a690eec4eeb"}]}