)]}'
{"swift/obj/reconstructor.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"235ddb77d131425be1a66897b6a48ed7573c3719","unresolved":false,"context_lines":[{"line_number":833,"context_line":"        for policy, local_devices in policy2devices.items():"},{"line_number":834,"context_line":"            df_mgr \u003d self._df_router[policy]"},{"line_number":835,"context_line":"            for local_dev in local_devices:"},{"line_number":836,"context_line":"                self.reconstruction_device_count +\u003d 1"},{"line_number":837,"context_line":"                dev_path \u003d df_mgr.get_dev_path(local_dev[\u0027device\u0027])"},{"line_number":838,"context_line":"                if not dev_path:"},{"line_number":839,"context_line":"                    self.logger.warning(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_ef2c0404","line":836,"updated":"2017-01-30 21:37:33.000000000","message":"Shouldn\u0027t this get moved out, too?\n\nWait, can it? Does this even *mean* anything anymore?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c4516108313cffc02f3297cdb818bb4fd9b62d1","unresolved":false,"context_lines":[{"line_number":833,"context_line":"        for policy, local_devices in policy2devices.items():"},{"line_number":834,"context_line":"            df_mgr \u003d self._df_router[policy]"},{"line_number":835,"context_line":"            for local_dev in local_devices:"},{"line_number":836,"context_line":"                self.reconstruction_device_count +\u003d 1"},{"line_number":837,"context_line":"                dev_path \u003d df_mgr.get_dev_path(local_dev[\u0027device\u0027])"},{"line_number":838,"context_line":"                if not dev_path:"},{"line_number":839,"context_line":"                    self.logger.warning(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_8d6b5db6","line":836,"in_reply_to":"3a461143_ec706742","updated":"2017-01-31 21:54:21.000000000","message":"IIUC, it was another way of looking at the completion percentage -- only now the dpercentage that gets logged in stats_line is always going to be 100% :-/\n\nShould we just drop that from the logging? It seems like it\u0027s no longer useful.","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"801279b9f1e2e1ea230c2c4e3b8613a61cf14c7e","unresolved":false,"context_lines":[{"line_number":833,"context_line":"        for policy, local_devices in policy2devices.items():"},{"line_number":834,"context_line":"            df_mgr \u003d self._df_router[policy]"},{"line_number":835,"context_line":"            for local_dev in local_devices:"},{"line_number":836,"context_line":"                self.reconstruction_device_count +\u003d 1"},{"line_number":837,"context_line":"                dev_path \u003d df_mgr.get_dev_path(local_dev[\u0027device\u0027])"},{"line_number":838,"context_line":"                if not dev_path:"},{"line_number":839,"context_line":"                    self.logger.warning(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_ec706742","line":836,"in_reply_to":"3a461143_ef2c0404","updated":"2017-01-31 19:55:45.000000000","message":"I *honestly* have no idea - and I *tried* to understand this guys stats - really did - but couldn\u0027t grok it :\u0027(","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"235ddb77d131425be1a66897b6a48ed7573c3719","unresolved":false,"context_lines":[{"line_number":842,"context_line":"                data_dir \u003d get_data_dir(policy)"},{"line_number":843,"context_line":"                obj_path \u003d join(dev_path, data_dir)"},{"line_number":844,"context_line":"                tmp_path \u003d join(dev_path, get_tmp_dir(int(policy)))"},{"line_number":845,"context_line":"                unlink_older_than(tmp_path, time.time() -"},{"line_number":846,"context_line":"                                  df_mgr.reclaim_age)"},{"line_number":847,"context_line":"                if not os.path.exists(obj_path):"},{"line_number":848,"context_line":"                    try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_8f6c6036","line":845,"range":{"start_line":845,"start_character":44,"end_line":845,"end_character":55},"updated":"2017-01-30 21:37:33.000000000","message":"Now that we\u0027re doing all of these together, would it make sense to take a single timestamp around where we initialize all_parts, and use that throughout? Honest question; I\u0027m not sure either is demonstrably *better* than the other.","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"801279b9f1e2e1ea230c2c4e3b8613a61cf14c7e","unresolved":false,"context_lines":[{"line_number":842,"context_line":"                data_dir \u003d get_data_dir(policy)"},{"line_number":843,"context_line":"                obj_path \u003d join(dev_path, data_dir)"},{"line_number":844,"context_line":"                tmp_path \u003d join(dev_path, get_tmp_dir(int(policy)))"},{"line_number":845,"context_line":"                unlink_older_than(tmp_path, time.time() -"},{"line_number":846,"context_line":"                                  df_mgr.reclaim_age)"},{"line_number":847,"context_line":"                if not os.path.exists(obj_path):"},{"line_number":848,"context_line":"                    try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_6c84770a","line":845,"in_reply_to":"3a461143_8f6c6036","updated":"2017-01-31 19:55:45.000000000","message":"acctually I think this should move out of this loop!  not sure how I missed it!","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"235ddb77d131425be1a66897b6a48ed7573c3719","unresolved":false,"context_lines":[{"line_number":864,"context_line":"                    if partition in (\u0027auditor_status_ALL.json\u0027,"},{"line_number":865,"context_line":"                                     \u0027auditor_status_ZBF.json\u0027):"},{"line_number":866,"context_line":"                        continue"},{"line_number":867,"context_line":"                    if not partition.isdigit():"},{"line_number":868,"context_line":"                        self.logger.warning("},{"line_number":869,"context_line":"                            \u0027Unexpected entity in data dir: %r\u0027 % part_path)"},{"line_number":870,"context_line":"                        remove_file(part_path)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_6fdd74df","line":867,"updated":"2017-01-30 21:37:33.000000000","message":"?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c4516108313cffc02f3297cdb818bb4fd9b62d1","unresolved":false,"context_lines":[{"line_number":864,"context_line":"                    if partition in (\u0027auditor_status_ALL.json\u0027,"},{"line_number":865,"context_line":"                                     \u0027auditor_status_ZBF.json\u0027):"},{"line_number":866,"context_line":"                        continue"},{"line_number":867,"context_line":"                    if not partition.isdigit():"},{"line_number":868,"context_line":"                        self.logger.warning("},{"line_number":869,"context_line":"                            \u0027Unexpected entity in data dir: %r\u0027 % part_path)"},{"line_number":870,"context_line":"                        remove_file(part_path)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_0dd3ed60","line":867,"in_reply_to":"3a461143_0c92134c","updated":"2017-01-31 21:54:21.000000000","message":"Ah, makes more sense now. So now we can return all-digit files (which shouldn\u0027t really happen in practice, but who knows), and later we deal with them in delete_partition, which we know *must* be the one that gets spawned because _get_part_jobs (and therefore build_reconstruction_jobs) will now return [] on ENOTDIR.\n\nShould we just get this out of here entirely then? Handle non-numerics up in _get_part_jobs and punt the logging/remove_file call to delete_partition regardless?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"801279b9f1e2e1ea230c2c4e3b8613a61cf14c7e","unresolved":false,"context_lines":[{"line_number":864,"context_line":"                    if partition in (\u0027auditor_status_ALL.json\u0027,"},{"line_number":865,"context_line":"                                     \u0027auditor_status_ZBF.json\u0027):"},{"line_number":866,"context_line":"                        continue"},{"line_number":867,"context_line":"                    if not partition.isdigit():"},{"line_number":868,"context_line":"                        self.logger.warning("},{"line_number":869,"context_line":"                            \u0027Unexpected entity in data dir: %r\u0027 % part_path)"},{"line_number":870,"context_line":"                        remove_file(part_path)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_0c92134c","line":867,"in_reply_to":"3a461143_6fdd74df","updated":"2017-01-31 19:55:45.000000000","message":"trying to avoid the stat call of isdir!  I want this loop to be tight and do as little io as possible!","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c4516108313cffc02f3297cdb818bb4fd9b62d1","unresolved":false,"context_lines":[{"line_number":868,"context_line":"                        self.logger.warning("},{"line_number":869,"context_line":"                            \u0027Unexpected entity in data dir: %r\u0027 % part_path)"},{"line_number":870,"context_line":"                        remove_file(part_path)"},{"line_number":871,"context_line":"                        self.reconstruction_part_count +\u003d 1"},{"line_number":872,"context_line":"                        continue"},{"line_number":873,"context_line":"                    partition \u003d int(partition)"},{"line_number":874,"context_line":"                    if override_partitions and (partition not in"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_3fa7ec0b","line":871,"updated":"2017-01-31 21:54:21.000000000","message":"I suppose this may front-load our completion percentages a bit, but I guess this path shouldn\u0027t generally be hit in a healthy, well-functioning cluster?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"235ddb77d131425be1a66897b6a48ed7573c3719","unresolved":false,"context_lines":[{"line_number":913,"context_line":"    def delete_partition(self, path):"},{"line_number":914,"context_line":"        self.logger.info(_(\"Removing partition: %s\"), path)"},{"line_number":915,"context_line":"        tpool.execute(shutil.rmtree, path, ignore_errors\u003dTrue)"},{"line_number":916,"context_line":"        remove_file(path)"},{"line_number":917,"context_line":""},{"line_number":918,"context_line":"    def reconstruct(self, **kwargs):"},{"line_number":919,"context_line":"        \"\"\"Run a reconstruction pass\"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_4fea78ad","line":916,"updated":"2017-01-30 21:37:33.000000000","message":"?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"801279b9f1e2e1ea230c2c4e3b8613a61cf14c7e","unresolved":false,"context_lines":[{"line_number":913,"context_line":"    def delete_partition(self, path):"},{"line_number":914,"context_line":"        self.logger.info(_(\"Removing partition: %s\"), path)"},{"line_number":915,"context_line":"        tpool.execute(shutil.rmtree, path, ignore_errors\u003dTrue)"},{"line_number":916,"context_line":"        remove_file(path)"},{"line_number":917,"context_line":""},{"line_number":918,"context_line":"    def reconstruct(self, **kwargs):"},{"line_number":919,"context_line":"        \"\"\"Run a reconstruction pass\"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_8cd443ec","line":916,"in_reply_to":"3a461143_4fea78ad","updated":"2017-01-31 19:55:45.000000000","message":"some test should catch this if you remove it - but I\u0027ll double check - IIRC it has something to do with the \"isdir\" call being moved into out of the collection loop","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c4516108313cffc02f3297cdb818bb4fd9b62d1","unresolved":false,"context_lines":[{"line_number":913,"context_line":"    def delete_partition(self, path):"},{"line_number":914,"context_line":"        self.logger.info(_(\"Removing partition: %s\"), path)"},{"line_number":915,"context_line":"        tpool.execute(shutil.rmtree, path, ignore_errors\u003dTrue)"},{"line_number":916,"context_line":"        remove_file(path)"},{"line_number":917,"context_line":""},{"line_number":918,"context_line":"    def reconstruct(self, **kwargs):"},{"line_number":919,"context_line":"        \"\"\"Run a reconstruction pass\"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_eda781a4","line":916,"in_reply_to":"3a461143_8cd443ec","updated":"2017-01-31 21:54:21.000000000","message":"Yup, makes sense now in the context of the isdigit/isdir discussion above. Although I suppose now the info logging isn\u0027t necessarily accurate... but it\u0027s rather unlikely to come up anyway, right?\n\nShould this be in a tpool, like the rmtree call?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"}],"test/unit/obj/test_reconstructor.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"235ddb77d131425be1a66897b6a48ed7573c3719","unresolved":false,"context_lines":[{"line_number":632,"context_line":"        matched \u003d stats_lines \u0026 acceptable"},{"line_number":633,"context_line":"        self.assertEqual(matched, acceptable,"},{"line_number":634,"context_line":"                         \u0027missing some expected acceptable:\\n%s\u0027 % ("},{"line_number":635,"context_line":"                             \u0027\\n\u0027.join(sorted(acceptable - matched))))"},{"line_number":636,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_device_count, 4)"},{"line_number":637,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_part_count, 8)"},{"line_number":638,"context_line":"        self.assertEqual(self.reconstructor.part_count, 8)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_8f7620d5","line":635,"updated":"2017-01-30 21:37:33.000000000","message":"I\u0027m kinda surprised we never make assertions about\n\n stats_lines - acceptable\n\n:-/","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"801279b9f1e2e1ea230c2c4e3b8613a61cf14c7e","unresolved":false,"context_lines":[{"line_number":632,"context_line":"        matched \u003d stats_lines \u0026 acceptable"},{"line_number":633,"context_line":"        self.assertEqual(matched, acceptable,"},{"line_number":634,"context_line":"                         \u0027missing some expected acceptable:\\n%s\u0027 % ("},{"line_number":635,"context_line":"                             \u0027\\n\u0027.join(sorted(acceptable - matched))))"},{"line_number":636,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_device_count, 4)"},{"line_number":637,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_part_count, 8)"},{"line_number":638,"context_line":"        self.assertEqual(self.reconstructor.part_count, 8)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_ecece7bc","line":635,"in_reply_to":"3a461143_8f7620d5","updated":"2017-01-31 19:55:45.000000000","message":"that would be ... other log lines that are not the specific patterns we\u0027re looking for?  how do sets work!?","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c4516108313cffc02f3297cdb818bb4fd9b62d1","unresolved":false,"context_lines":[{"line_number":632,"context_line":"        matched \u003d stats_lines \u0026 acceptable"},{"line_number":633,"context_line":"        self.assertEqual(matched, acceptable,"},{"line_number":634,"context_line":"                         \u0027missing some expected acceptable:\\n%s\u0027 % ("},{"line_number":635,"context_line":"                             \u0027\\n\u0027.join(sorted(acceptable - matched))))"},{"line_number":636,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_device_count, 4)"},{"line_number":637,"context_line":"        self.assertEqual(self.reconstructor.reconstruction_part_count, 8)"},{"line_number":638,"context_line":"        self.assertEqual(self.reconstructor.part_count, 8)"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a461143_0d020d0a","line":635,"in_reply_to":"3a461143_ecece7bc","updated":"2017-01-31 21:54:21.000000000","message":"Given the variable name, I was assuming it means \"unacceptable stats_lines\".","commit_id":"2f0ab78f9ff85483a157c9cbb17b50eeff539ef9"}]}