)]}'
{"swift/cli/relinker.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":17,"context_line":"import fcntl"},{"line_number":18,"context_line":"import logging"},{"line_number":19,"context_line":"import os"},{"line_number":20,"context_line":"import pickle"},{"line_number":21,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":22,"context_line":"from swift.common.exceptions import DiskFileDeleted, DiskFileNotExist, \\"},{"line_number":23,"context_line":"    DiskFileQuarantined"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_00d857ea","line":20,"updated":"2020-01-28 20:31:10.000000000","message":"IDK about increasing our use of pickle -- would json be better? We already str() the keys anyway...","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":17,"context_line":"import fcntl"},{"line_number":18,"context_line":"import logging"},{"line_number":19,"context_line":"import os"},{"line_number":20,"context_line":"import pickle"},{"line_number":21,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":22,"context_line":"from swift.common.exceptions import DiskFileDeleted, DiskFileNotExist, \\"},{"line_number":23,"context_line":"    DiskFileQuarantined"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_6c1d54bc","line":20,"in_reply_to":"3fa7e38b_00d857ea","updated":"2020-03-27 22:05:11.000000000","message":"Done","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":43,"context_line":"        # This is not a cleanup after cancel, partitions in the upper half are"},{"line_number":44,"context_line":"        # new partitions, there is nothing to relink/cleanup from there"},{"line_number":45,"context_line":"        partitions \u003d [p for p in partitions"},{"line_number":46,"context_line":"                      if int(p) \u003c 2 ** next_part_power / 2]"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"    # Format: { \u0027part\u0027: [relinked, cleaned] }"},{"line_number":49,"context_line":"    if os.path.exists(state_file):"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_a075c301","line":46,"updated":"2020-01-28 20:31:10.000000000","message":"++","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":50,"context_line":"        with open(state_file, \u0027a\u0027) as lock:"},{"line_number":51,"context_line":"            fcntl.flock(lock.fileno(), fcntl.LOCK_EX)"},{"line_number":52,"context_line":"            with open(state_file, \u0027rb\u0027) as f:"},{"line_number":53,"context_line":"                states \u003d pickle.load(f)"},{"line_number":54,"context_line":"            missing \u003d list(set(partitions) - set(states.keys()))"},{"line_number":55,"context_line":"            if missing:"},{"line_number":56,"context_line":"                # All missing partitions was created after the first run of"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_00f13764","line":53,"updated":"2020-01-28 20:31:10.000000000","message":"What if the file got corrupted? I suppose, just rm the file and assume fresh state.\n\nI\u0027m not sure we\u0027d have enough context to tell which device it was on just from the traceback, though...","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":50,"context_line":"        with open(state_file, \u0027a\u0027) as lock:"},{"line_number":51,"context_line":"            fcntl.flock(lock.fileno(), fcntl.LOCK_EX)"},{"line_number":52,"context_line":"            with open(state_file, \u0027rb\u0027) as f:"},{"line_number":53,"context_line":"                states \u003d pickle.load(f)"},{"line_number":54,"context_line":"            missing \u003d list(set(partitions) - set(states.keys()))"},{"line_number":55,"context_line":"            if missing:"},{"line_number":56,"context_line":"                # All missing partitions was created after the first run of"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_ac7c7ceb","line":53,"in_reply_to":"3fa7e38b_00f13764","updated":"2020-03-27 22:05:11.000000000","message":"Done","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":67,"context_line":"                partitions \u003d [str(p) for p, (r, c) in states.items() if not c]"},{"line_number":68,"context_line":"    else:"},{"line_number":69,"context_line":"        states \u003d {str(p): [False, False] for p in partitions}"},{"line_number":70,"context_line":"        with open(state_file, \u0027wb\u0027) as f:"},{"line_number":71,"context_line":"            pickle.dump(states, f)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"    # Always scan the partitions in reverse order to minimize the amount of IO"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_9d027071","line":70,"updated":"2020-01-28 20:31:10.000000000","message":"Should we lock the file here, too?","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":67,"context_line":"                partitions \u003d [str(p) for p, (r, c) in states.items() if not c]"},{"line_number":68,"context_line":"    else:"},{"line_number":69,"context_line":"        states \u003d {str(p): [False, False] for p in partitions}"},{"line_number":70,"context_line":"        with open(state_file, \u0027wb\u0027) as f:"},{"line_number":71,"context_line":"            pickle.dump(states, f)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"    # Always scan the partitions in reverse order to minimize the amount of IO"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_8c49b8d1","line":70,"in_reply_to":"3fa7e38b_9d027071","updated":"2020-03-27 22:05:11.000000000","message":"Now handled more globally (lock the device before working on it)","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":97,"context_line":"        with open(state_file, \u0027a\u0027) as lock:"},{"line_number":98,"context_line":"            fcntl.flock(lock.fileno(), fcntl.LOCK_EX)"},{"line_number":99,"context_line":"            with open(state_file, \u0027rb\u0027) as f:"},{"line_number":100,"context_line":"                states \u003d pickle.load(f)"},{"line_number":101,"context_line":"            if step \u003d\u003d STEP_RELINK:"},{"line_number":102,"context_line":"                states[part][0] \u003d True"},{"line_number":103,"context_line":"            elif step \u003d\u003d STEP_CLEANUP:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_c0947fe9","line":100,"updated":"2020-01-28 20:31:10.000000000","message":"Another opportunity for an unpickling error.","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":97,"context_line":"        with open(state_file, \u0027a\u0027) as lock:"},{"line_number":98,"context_line":"            fcntl.flock(lock.fileno(), fcntl.LOCK_EX)"},{"line_number":99,"context_line":"            with open(state_file, \u0027rb\u0027) as f:"},{"line_number":100,"context_line":"                states \u003d pickle.load(f)"},{"line_number":101,"context_line":"            if step \u003d\u003d STEP_RELINK:"},{"line_number":102,"context_line":"                states[part][0] \u003d True"},{"line_number":103,"context_line":"            elif step \u003d\u003d STEP_CLEANUP:"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_6c4434b8","line":100,"in_reply_to":"3fa7e38b_c0947fe9","updated":"2020-03-27 22:05:11.000000000","message":"Now the state file is only read once","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"        def devices_filter(_, devices):"},{"line_number":129,"context_line":"            if device:"},{"line_number":130,"context_line":"                return list(set([device]).intersection(set(devices)))"},{"line_number":131,"context_line":"            else:"},{"line_number":132,"context_line":"                return devices"},{"line_number":133,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_9d1030ad","line":130,"updated":"2020-01-28 20:31:10.000000000","message":"Would\n\n [d for d in devices if d \u003d\u003d device]\n\nbe more clear? *shrug*","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"        def devices_filter(_, devices):"},{"line_number":129,"context_line":"            if device:"},{"line_number":130,"context_line":"                return list(set([device]).intersection(set(devices)))"},{"line_number":131,"context_line":"            else:"},{"line_number":132,"context_line":"                return devices"},{"line_number":133,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_ec8684db","line":130,"in_reply_to":"3fa7e38b_9d1030ad","updated":"2020-03-27 22:05:11.000000000","message":"Done","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":131,"context_line":"            else:"},{"line_number":132,"context_line":"                return devices"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"        def relink_partition_filter(datadir_path, partitions):"},{"line_number":135,"context_line":"            return partitions_filter(STEP_RELINK, part_power, next_part_power,"},{"line_number":136,"context_line":"                                     datadir_path, partitions)"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"        def relink_hook_post_partition(partition_path):"},{"line_number":139,"context_line":"            return hook_post_partition(STEP_RELINK, partition_path)"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"        locations \u003d audit_location_generator("},{"line_number":142,"context_line":"            devices,"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_1d24400d","line":139,"range":{"start_line":134,"start_character":8,"end_line":139,"end_character":67},"updated":"2020-01-28 20:31:10.000000000","message":"Maybe use functools.partial?","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":131,"context_line":"            else:"},{"line_number":132,"context_line":"                return devices"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"        def relink_partition_filter(datadir_path, partitions):"},{"line_number":135,"context_line":"            return partitions_filter(STEP_RELINK, part_power, next_part_power,"},{"line_number":136,"context_line":"                                     datadir_path, partitions)"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"        def relink_hook_post_partition(partition_path):"},{"line_number":139,"context_line":"            return hook_post_partition(STEP_RELINK, partition_path)"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"        locations \u003d audit_location_generator("},{"line_number":142,"context_line":"            devices,"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_4c6df030","line":139,"range":{"start_line":134,"start_character":8,"end_line":139,"end_character":67},"in_reply_to":"3fa7e38b_1d24400d","updated":"2020-03-27 22:05:11.000000000","message":"Done","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"304c253c4299ebaaec3cdf480d838cbe83dbcbda","unresolved":false,"context_lines":[{"line_number":46,"context_line":"    lock_file \u003d os.path.join(device_path, LOCK_FILE.format(datadir\u003ddatadir))"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"    fd \u003d os.open(lock_file, os.O_CREAT | os.O_WRONLY)"},{"line_number":49,"context_line":"    fcntl.flock(fd, fcntl.LOCK_EX)"},{"line_number":50,"context_line":"    locks[0] \u003d fd"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"    state_file \u003d os.path.join(device_path, STATE_FILE.format(datadir\u003ddatadir))"}],"source_content_type":"text/x-python","patch_set":9,"id":"ff570b3c_8ffce972","line":49,"updated":"2020-06-02 06:24:44.000000000","message":"Should we include a fcntl.LOCK_NB like the tests do? I\u0027m not sure that waiting for whatever other relinker process to be done is terribly useful; I would\u0027ve thought just knowing that some other process is try to handle it would be sufficient information.","commit_id":"3061ec803f1bf11ba88d8eaec96db2c007307c3a"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":3139,"context_line":"                             hook_pre_device\u003dNone, hook_post_device\u003dNone,"},{"line_number":3140,"context_line":"                             hook_pre_partition\u003dNone, hook_post_partition\u003dNone,"},{"line_number":3141,"context_line":"                             hook_pre_suffix\u003dNone, hook_post_suffix\u003dNone,"},{"line_number":3142,"context_line":"                             hook_pre_hash\u003dNone, hook_post_hash\u003dNone):"},{"line_number":3143,"context_line":"    \"\"\""},{"line_number":3144,"context_line":"    Given a devices path and a data directory, yield (path, device,"},{"line_number":3145,"context_line":"    partition) for all files in that directory"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_ff9dc308","line":3142,"updated":"2020-01-28 20:31:10.000000000","message":"I wonder if this would be better as filter and hooks args, each of which is a dict....\n\nRegardless, we probably ought to update the docstring.","commit_id":"b335b145b6d288cfecd7fee25a65bef2e0e64e61"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":3209,"context_line":"                    except OSError as e:"},{"line_number":3210,"context_line":"                        if e.errno !\u003d errno.ENOTDIR:"},{"line_number":3211,"context_line":"                            raise"},{"line_number":3212,"context_line":"                        continue"},{"line_number":3213,"context_line":"                    for fname in files:"},{"line_number":3214,"context_line":"                        if suffix and not fname.endswith(suffix):"},{"line_number":3215,"context_line":"                            continue"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_3f66fb2a","line":3212,"range":{"start_line":3212,"start_character":24,"end_line":3212,"end_character":32},"updated":"2020-01-28 20:31:10.000000000","message":"Hmm... so if any of these continues fire, we get a pre hook called but no post hook?","commit_id":"b335b145b6d288cfecd7fee25a65bef2e0e64e61"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a2d7bf2943ad1a2ae48de71b7433e0203ea26dfa","unresolved":false,"context_lines":[{"line_number":3168,"context_line":"    :suffixes_filter: a callable taking (part_path, [list of suffixes]) as"},{"line_number":3169,"context_line":"                      parameters and returning a [list of suffixes]"},{"line_number":3170,"context_line":"    :hashes_filter: a callable taking (suff_path, [list of hashes]) as"},{"line_number":3171,"context_line":"                    parameters and returning a [list of hashes]"},{"line_number":3172,"context_line":"    :hook_pre_device: a callable taking device_path as parameter"},{"line_number":3173,"context_line":"    :hook_post_device: a callable taking device_path as parameter"},{"line_number":3174,"context_line":"    :hook_pre_partition: a callable taking part_path as parameter"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa7e38b_20523369","line":3171,"updated":"2020-01-28 20:31:10.000000000","message":"If we wanted, we could use hashes_filter when identifying what needs relinking -- it\u0027d save us a listdir, anyway.","commit_id":"21f031015e0394c803059b7782352a1a77063b01"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"a4c8d9a818dfbdd7b3dd617a401c6467ebedddae","unresolved":false,"context_lines":[{"line_number":3168,"context_line":"    :suffixes_filter: a callable taking (part_path, [list of suffixes]) as"},{"line_number":3169,"context_line":"                      parameters and returning a [list of suffixes]"},{"line_number":3170,"context_line":"    :hashes_filter: a callable taking (suff_path, [list of hashes]) as"},{"line_number":3171,"context_line":"                    parameters and returning a [list of hashes]"},{"line_number":3172,"context_line":"    :hook_pre_device: a callable taking device_path as parameter"},{"line_number":3173,"context_line":"    :hook_post_device: a callable taking device_path as parameter"},{"line_number":3174,"context_line":"    :hook_pre_partition: a callable taking part_path as parameter"}],"source_content_type":"text/x-python","patch_set":5,"id":"df33271e_2c686c1f","line":3171,"in_reply_to":"3fa7e38b_20523369","updated":"2020-03-27 22:05:11.000000000","message":"Actually, for relinking step, everything needs relink expect partition 0, so the win is small; But for cleaning it could be significant. Done.","commit_id":"21f031015e0394c803059b7782352a1a77063b01"}],"test/unit/cli/test_relinker.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"304c253c4299ebaaec3cdf480d838cbe83dbcbda","unresolved":false,"context_lines":[{"line_number":172,"context_line":""},{"line_number":173,"context_line":"    def test_relink_cleanup(self):"},{"line_number":174,"context_line":"        state_file \u003d os.path.join(self.devices, self.existing_device,"},{"line_number":175,"context_line":"                                  \u0027relink.objects.json\u0027)"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"        self.rb.prepare_increase_partition_power()"},{"line_number":178,"context_line":"        self._save_ring()"}],"source_content_type":"text/x-python","patch_set":9,"id":"ff570b3c_8fe349d8","line":175,"updated":"2020-06-02 06:24:44.000000000","message":"Worth a sanity check like\n\n self.assertFalse(os.path.exists(state_file))\n\n?","commit_id":"3061ec803f1bf11ba88d8eaec96db2c007307c3a"}]}