)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"29ebb5649282bce5e91d933ec3a05f02543dfd52","unresolved":false,"context_lines":[{"line_number":11,"context_line":""},{"line_number":12,"context_line":"If we truncate invalidations we must write down hashes.pkl; the best way"},{"line_number":13,"context_line":"to represent \"every suffix is invalid please do_listdir\" is to write"},{"line_number":14,"context_line":"down the empty dict with the updated key but still return None."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Change-Id: I08c8cf09282f737103e580c1f57923b399abe58c"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"3a461143_d0fb4777","line":14,"updated":"2017-01-31 15:11:44.000000000","message":"None is no longer returned","commit_id":"856ef641ac0f7d870e7a1515065237f882c86fb1"},{"author":{"_account_id":12261,"name":"Mahati","email":"mahati.chamarthy@gmail.com","username":"Mahati"},"change_message_id":"26b8194256b911948c89eec6e7031e250a766339","unresolved":false,"context_lines":[{"line_number":11,"context_line":"partition with no hashes.pkl"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"If we truncate invalidations we must write down hashes.pkl with"},{"line_number":14,"context_line":"updated key set to current time and the valid key set to False."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Change-Id: I08c8cf09282f737103e580c1f57923b399abe58c"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"1a1ced50_a67d31b3","line":14,"updated":"2017-03-20 11:47:33.000000000","message":"Commit message must change to reflect the patchset. This patch no longer \"updates key\" - it has been done in the parent change: https://review.openstack.org/#/c/419787\n\nThis patch removes the hashes file existence check which is no longer necessary after parent change - I\u0027m confident you can reword it *much better* ;)","commit_id":"3cc97b5d30fe034a9b1d18a43eda9db6e1eaa4ac"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3dc65f414c6901f127038d47bec2f32e2e197f7","unresolved":false,"context_lines":[{"line_number":267,"context_line":""},{"line_number":268,"context_line":"    The updated key is added to hashes before it is written."},{"line_number":269,"context_line":"    \"\"\""},{"line_number":270,"context_line":"    hashes_file \u003d join(partition_dir, HASH_FILE)"},{"line_number":271,"context_line":"    hashes[\u0027updated\u0027] \u003d time.time()"},{"line_number":272,"context_line":"    return write_pickle(hashes, hashes_file, partition_dir, PICKLE_PROTOCOL)"},{"line_number":273,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"3a461143_b078d72c","line":270,"updated":"2017-01-27 19:39:53.000000000","message":"reading the comments on the previous patch more closely I\u0027m starting to think this code should just:\n\n    hashes \u003d hashes or {}","commit_id":"d2c7c807847cf903911e6d62624ff486333fcb9a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dfcb61bbafb0984666aac45160ba893d734e49bf","unresolved":false,"context_lines":[{"line_number":289,"context_line":"    invalidations_file \u003d join(partition_dir, HASH_INVALIDATIONS_FILE)"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"    if not os.path.exists(hashes_file):"},{"line_number":292,"context_line":"        if os.path.exists(invalidations_file):"},{"line_number":293,"context_line":"            # no hashes at all -\u003e everything\u0027s invalid, so empty the file with"},{"line_number":294,"context_line":"            # the invalid suffixes in it, if it exists"},{"line_number":295,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3a461143_30bf2bab","side":"PARENT","line":292,"updated":"2017-01-31 15:11:08.000000000","message":"this was prone to a race:\n\n  process X calls consolidate_hashes, no hashes.pkl, process X begins to hash suffixes.hashes.\n  Process Y invalidates a suffix and appends to hashes.invalid.\n  Process Z calls consolidate_hashes, no hashes.pkl, truncates hashes.invalid.\n  Process X completes suffix hashing unaware of recent invalidation and writes out of data hashes.pkl.","commit_id":"a6ee421ab5c84bf2a5076ad4deec8881d7cc34c6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f4974ff1a9be237ff1c410f7862855aab56da047","unresolved":false,"context_lines":[{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        if orig_hashes is None:"},{"line_number":1073,"context_line":"            # consolidate_hashes returns None if hashes.pkl does not exist"},{"line_number":1074,"context_line":"            orig_hashes \u003d {\u0027valid\u0027: False}"},{"line_number":1075,"context_line":"        if not orig_hashes[\u0027valid\u0027]:"},{"line_number":1076,"context_line":"            # This is the only path to a valid hashes from invalid read (e.g."},{"line_number":1077,"context_line":"            # does not exist, corrupt, etc.).  Moreover, in order to write this"}],"source_content_type":"text/x-python","patch_set":5,"id":"3a461143_973d31df","side":"PARENT","line":1074,"updated":"2017-01-31 17:09:48.000000000","message":"so much better!","commit_id":"a6ee421ab5c84bf2a5076ad4deec8881d7cc34c6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":297,"context_line":"                    pass"},{"line_number":298,"context_line":"            except OSError as e:"},{"line_number":299,"context_line":"                if e.errno !\u003d errno.ENOENT:"},{"line_number":300,"context_line":"                    raise"},{"line_number":301,"context_line":"        return None"},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"    with lock_path(partition_dir):"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_db0e198c","side":"PARENT","line":300,"updated":"2017-04-07 23:35:32.000000000","message":"Yeah i\u0027m glad to see *this* go - it\u0027s a very small race, but still pretty scary!","commit_id":"fa456a590c241d4cdfdccaec95466262029fb7d7"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        if orig_hashes is None:"},{"line_number":1073,"context_line":"            # consolidate_hashes returns None if hashes.pkl does not exist"},{"line_number":1074,"context_line":"            orig_hashes \u003d {\u0027valid\u0027: False}"},{"line_number":1075,"context_line":"        if not orig_hashes[\u0027valid\u0027]:"},{"line_number":1076,"context_line":"            # This is the only path to a valid hashes from invalid read (e.g."},{"line_number":1077,"context_line":"            # does not exist, corrupt, etc.).  Moreover, in order to write this"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_fb7c3dec","side":"PARENT","line":1074,"updated":"2017-04-07 23:35:32.000000000","message":"ok, so we *should* still come out of conslidate_hashes with an invalid entry.  Do we have a test that asserts do_listdir on first hashes.pkl even if we found invalidations, basically will we catch this regression:\n\n    diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py\n    index abc51dc..bcbc766 100644\n    --- a/swift/obj/diskfile.py\n    +++ b/swift/obj/diskfile.py\n    @@ -36,6 +36,7 @@ import errno\n     import fcntl\n     import json\n     import os\n    +orig_exists \u003d os.path.exists\n     import re\n     import time\n     import uuid\n    @@ -291,6 +292,11 @@ def consolidate_hashes(partition_dir):\n         with lock_path(partition_dir):\n             hashes \u003d read_hashes(partition_dir)\n     \n    +        if orig_exists(join(partition_dir, HASH_FILE)):\n    +            started_there \u003d True\n    +        else:\n    +            started_there \u003d False\n    +\n             found_invalidation_entry \u003d False\n             try:\n                 with open(invalidations_file, \u0027rb\u0027) as inv_fh:\n    @@ -303,6 +309,8 @@ def consolidate_hashes(partition_dir):\n                     raise\n     \n             if found_invalidation_entry:\n    +            if not started_there:\n    +                hashes[\u0027valid\u0027] \u003d True\n                 write_hashes(partition_dir, hashes)\n                 # Now that all the invalidations are reflected in hashes.pkl, it\u0027s\n                 # safe to clear out the invalidations file.\n\n\nHrmm... not *on purpose*\n\nhttps://gist.github.com/clayg/ed251962d9a5ddaa7276b452577cbf86","commit_id":"fa456a590c241d4cdfdccaec95466262029fb7d7"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":270,"context_line":"    hashes_file \u003d join(partition_dir, HASH_FILE)"},{"line_number":271,"context_line":"    # \u0027valid\u0027 key should always be set by the caller; however, if there\u0027s a bug"},{"line_number":272,"context_line":"    # setting invalid is most safe"},{"line_number":273,"context_line":"    hashes.setdefault(\u0027valid\u0027, False)"},{"line_number":274,"context_line":"    hashes[\u0027updated\u0027] \u003d time.time()"},{"line_number":275,"context_line":"    write_pickle(hashes, hashes_file, partition_dir, PICKLE_PROTOCOL)"},{"line_number":276,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_7b514d62","line":273,"updated":"2017-04-07 23:35:32.000000000","message":"oh, i forgot you can *write* invalid hashes!","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":303,"context_line":"                raise"},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"        if found_invalidation_entry:"},{"line_number":306,"context_line":"            write_hashes(partition_dir, hashes)"},{"line_number":307,"context_line":"            # Now that all the invalidations are reflected in hashes.pkl, it\u0027s"},{"line_number":308,"context_line":"            # safe to clear out the invalidations file."},{"line_number":309,"context_line":"            with open(invalidations_file, \u0027wb\u0027) as inv_fh:"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_5b220902","line":306,"updated":"2017-04-07 23:35:32.000000000","message":"so we\u0027re going to come out of this still \"invalid\" hashes - we added all he invalidations - so ... we could maybe skip the listdir?","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"522a44bb5ea9c4f3c197adf91aea0b510eb1c95e","unresolved":false,"context_lines":[{"line_number":303,"context_line":"                raise"},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"        if found_invalidation_entry:"},{"line_number":306,"context_line":"            write_hashes(partition_dir, hashes)"},{"line_number":307,"context_line":"            # Now that all the invalidations are reflected in hashes.pkl, it\u0027s"},{"line_number":308,"context_line":"            # safe to clear out the invalidations file."},{"line_number":309,"context_line":"            with open(invalidations_file, \u0027wb\u0027) as inv_fh:"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_9c2632ce","line":306,"in_reply_to":"bff0334d_5b220902","updated":"2017-04-10 09:37:40.000000000","message":"We should keep listdir to find files recieved from replicator - because hash recalculation after rsync is not guaranteed. It is not necessary but it is better.","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":325,"context_line":"    invalidations_file \u003d join(partition_dir, HASH_INVALIDATIONS_FILE)"},{"line_number":326,"context_line":"    with lock_path(partition_dir):"},{"line_number":327,"context_line":"        with open(invalidations_file, \u0027ab\u0027) as inv_fh:"},{"line_number":328,"context_line":"            inv_fh.write(suffix + \"\\n\")"},{"line_number":329,"context_line":""},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"class AuditLocation(object):"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_7b682d20","line":328,"updated":"2017-04-07 23:35:32.000000000","message":"no fast path for missing hashes.pkl - I guess we can trust the entires to be in the invalidations file...","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":1061,"context_line":"            # This is the only path to a valid hashes from invalid read (e.g."},{"line_number":1062,"context_line":"            # does not exist, corrupt, etc.).  Moreover, in order to write this"},{"line_number":1063,"context_line":"            # valid hashes we must read *the exact same* invalid state or we\u0027ll"},{"line_number":1064,"context_line":"            # trigger race detection."},{"line_number":1065,"context_line":"            do_listdir \u003d True"},{"line_number":1066,"context_line":"            hashes \u003d {\u0027valid\u0027: True}"},{"line_number":1067,"context_line":"            # If the exception handling around consolidate_hashes fired we\u0027re"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_2d182c73","line":1064,"updated":"2017-04-07 23:35:32.000000000","message":"this is a *great* invariant, as long as no hashes.pkl comes out of conslidate_hashes with valid \u003d False (which it does) we can be sure we will do_listdir (which I think is very much what we want!)","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9e6a107cc4546f617f1fd277139ed022db5943eb","unresolved":false,"context_lines":[{"line_number":6275,"context_line":"                    df.delete(self.ts())"},{"line_number":6276,"context_line":"                    # simulate another process calling get_hashes but failing"},{"line_number":6277,"context_line":"                    # after hash invalidation have been consolidated"},{"line_number":6278,"context_line":"                    df_mgr.consolidate_hashes(part_dir)"},{"line_number":6279,"context_line":"                    # get the updated suffix hash..."},{"line_number":6280,"context_line":"                    non_local[\u0027hash\u0027] \u003d orig_hash_suffix(suffix_dir)"},{"line_number":6281,"context_line":"                return result"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a461143_77166956","line":6278,"updated":"2017-01-27 18:30:32.000000000","message":"if there is no hashes.pkl then this call erases the suffix invalidation we just wrote to hashes.invalid :/","commit_id":"b13e9fcad941cdcf5a6ba18735ed30d520d93573"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"70c1714109a54ff47498a5ea8b1b38fa0e1be710","unresolved":false,"context_lines":[{"line_number":6280,"context_line":"                    if existing:"},{"line_number":6281,"context_line":"                        self.assertTrue(hashes[\u0027valid\u0027])"},{"line_number":6282,"context_line":"                    else:"},{"line_number":6283,"context_line":"                        self.assertFalse(hashes[\u0027valid\u0027])"},{"line_number":6284,"context_line":"                    # get the updated suffix hash..."},{"line_number":6285,"context_line":"                    non_local[\u0027hash\u0027] \u003d orig_hash_suffix(suffix_dir)"},{"line_number":6286,"context_line":"                return result"}],"source_content_type":"text/x-python","patch_set":7,"id":"bff0334d_3bf9854c","line":6283,"updated":"2017-04-07 23:35:32.000000000","message":"nice","commit_id":"52a23ddb3c1cbc41daca7b8238d43a9851641834"}]}