)]}'
{"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":832,"context_line":"        return cmp(self.internal, other.internal)"},{"line_number":833,"context_line":""},{"line_number":834,"context_line":"    def __hash__(self):"},{"line_number":835,"context_line":"        return hash(self.internal)"},{"line_number":836,"context_line":""},{"line_number":837,"context_line":""},{"line_number":838,"context_line":"def normalize_timestamp(timestamp):"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_34f2ba07","line":835,"updated":"2015-12-01 01:13:08.000000000","message":"nice","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"}],"swift/obj/diskfile.py":[{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":553,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":554,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":555,"context_line":"            except (DiskFileError, ValueError):"},{"line_number":556,"context_line":"                # ignore unexpected files"},{"line_number":557,"context_line":"                pass"},{"line_number":558,"context_line":"        for ext in exts:"},{"line_number":559,"context_line":"            # For each extension sort files into reverse chronological order."}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_aac33eec","line":556,"updated":"2015-11-10 22:34:51.000000000","message":"should we log any kind of warning?  under what conditions would an unexpected file be, well, expected?","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b57dedcee67e8e53974ab4d7e889a4e0ef4c491f","unresolved":false,"context_lines":[{"line_number":553,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":554,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":555,"context_line":"            except (DiskFileError, ValueError):"},{"line_number":556,"context_line":"                # ignore unexpected files"},{"line_number":557,"context_line":"                pass"},{"line_number":558,"context_line":"        for ext in exts:"},{"line_number":559,"context_line":"            # For each extension sort files into reverse chronological order."}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_3a454eb2","line":556,"in_reply_to":"9a8ffd7b_09f9614d","updated":"2015-11-26 16:28:09.000000000","message":"Follow-up - on master the use of ext \u003d splitext(afile)[1] means that any file that does not have an expected extension, or has no extension, will be silently ignored. See line 2027 in the base version of this review.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":553,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":554,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":555,"context_line":"            except (DiskFileError, ValueError):"},{"line_number":556,"context_line":"                # ignore unexpected files"},{"line_number":557,"context_line":"                pass"},{"line_number":558,"context_line":"        for ext in exts:"},{"line_number":559,"context_line":"            # For each extension sort files into reverse chronological order."}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_09f9614d","line":556,"in_reply_to":"da85f559_aac33eec","updated":"2015-11-25 19:18:13.000000000","message":"I don\u0027t believe we log anything at the moment. I guess I am being defensive, in that if someone did touch a junk file in an obj dir, swift would keep working and not blow up.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"74debb9d4eaaa6ba452afaf9f916a69008e3e24c","unresolved":false,"context_lines":[{"line_number":574,"context_line":"            exts[ext] \u003d exts[ext][:1]"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"        # delegate to subclass handler"},{"line_number":577,"context_line":"        self._gather_on_disk_file(exts, results, **kwargs)"},{"line_number":578,"context_line":""},{"line_number":579,"context_line":"        # set final choice of files"},{"line_number":580,"context_line":"        if exts[\u0027.ts\u0027]:"}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_451a987e","line":577,"updated":"2015-11-11 22:42:22.000000000","message":"I wonder if there\u0027s a better name for this function.  Looking at the set of calls required to process a dir its still a little confusing and I\u0027m thinking better names might help.  Maybe something like _further_process_files() or something... anything other than the same name as the calling function but singular instead of plural :)\n\nexample of confusing call tree based on current func names: \n_get_ondisk_file()---\u003eget_ondisk_files()--\u003egather_ondisk_files--\u003e_gather_ondisk_file()","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":574,"context_line":"            exts[ext] \u003d exts[ext][:1]"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"        # delegate to subclass handler"},{"line_number":577,"context_line":"        self._gather_on_disk_file(exts, results, **kwargs)"},{"line_number":578,"context_line":""},{"line_number":579,"context_line":"        # set final choice of files"},{"line_number":580,"context_line":"        if exts[\u0027.ts\u0027]:"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_69134d67","line":577,"in_reply_to":"da85f559_451a987e","updated":"2015-11-25 19:18:13.000000000","message":"agree","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":2048,"context_line":"            \u0027ext\u0027: ext,"},{"line_number":2049,"context_line":"        }"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"    def _gather_on_disk_file(self, exts, results, **kwargs):"},{"line_number":2052,"context_line":"        \"\"\""},{"line_number":2053,"context_line":"        Implement replication policy specific handling of .data files."},{"line_number":2054,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_0d4e243b","line":2051,"updated":"2015-11-10 22:34:51.000000000","message":"oh wow","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"74debb9d4eaaa6ba452afaf9f916a69008e3e24c","unresolved":false,"context_lines":[{"line_number":2217,"context_line":"            return dict([(ts, [info[\u0027frag_index\u0027] for info in frag_set])"},{"line_number":2218,"context_line":"                         for ts, frag_set in frag_sets.items()])"},{"line_number":2219,"context_line":""},{"line_number":2220,"context_line":"    def _get_ondisk_file(self, files):"},{"line_number":2221,"context_line":"        \"\"\""},{"line_number":2222,"context_line":"        The only difference between this method and the replication policy"},{"line_number":2223,"context_line":"        DiskFile method is passing in the frag_index kwarg to our manager\u0027s"}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_00e55eda","line":2220,"updated":"2015-11-11 22:42:22.000000000","message":"same comment wrt function name as I make further below.  Looking at the call tree we have this pattern in two places where there\u0027s a function (singular) that calls another names the same but plural.  Can we come up with something better here as well instead of _get_ondisk_file() and get_ondisk_files()? At a minimim make them both plural?  I dunno... I\u0027m getting too confused too easily these days :)","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":2217,"context_line":"            return dict([(ts, [info[\u0027frag_index\u0027] for info in frag_set])"},{"line_number":2218,"context_line":"                         for ts, frag_set in frag_sets.items()])"},{"line_number":2219,"context_line":""},{"line_number":2220,"context_line":"    def _get_ondisk_file(self, files):"},{"line_number":2221,"context_line":"        \"\"\""},{"line_number":2222,"context_line":"        The only difference between this method and the replication policy"},{"line_number":2223,"context_line":"        DiskFile method is passing in the frag_index kwarg to our manager\u0027s"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_091de1bc","line":2220,"in_reply_to":"da85f559_00e55eda","updated":"2015-11-25 19:18:13.000000000","message":"Yes I agree about the function names. I\u0027ll give it some thought.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":2347,"context_line":"        :param frag_index: if set, search for a specific fragment index .data"},{"line_number":2348,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2349,"context_line":"        \"\"\""},{"line_number":2350,"context_line":"        # First mark everything older than most recent .durable as obsolete"},{"line_number":2351,"context_line":"        # and remove from the exts dict."},{"line_number":2352,"context_line":"        for ext in exts.keys():"},{"line_number":2353,"context_line":"            exts[ext], older \u003d self._split_by_time("}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_8d2e9441","line":2350,"updated":"2015-11-10 22:34:51.000000000","message":"so much better than before","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":468,"context_line":"        :param filename: a string representing a filename"},{"line_number":469,"context_line":"        :param other_filename: a string representing a filename"},{"line_number":470,"context_line":"        :returns: True if filename is considered older, False otherwise."},{"line_number":471,"context_line":"        \"\"\""},{"line_number":472,"context_line":"        if not other_filename:"},{"line_number":473,"context_line":"            return False"},{"line_number":474,"context_line":"        ts \u003d self.parse_on_disk_filename(filename)[\u0027timestamp\u0027]"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_cff5decb","line":471,"updated":"2015-12-01 01:13:08.000000000","message":"seems like a great function - is a caller for it added later?  or did i miss something?","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":468,"context_line":"        :param filename: a string representing a filename"},{"line_number":469,"context_line":"        :param other_filename: a string representing a filename"},{"line_number":470,"context_line":"        :returns: True if filename is considered older, False otherwise."},{"line_number":471,"context_line":"        \"\"\""},{"line_number":472,"context_line":"        if not other_filename:"},{"line_number":473,"context_line":"            return False"},{"line_number":474,"context_line":"        ts \u003d self.parse_on_disk_filename(filename)[\u0027timestamp\u0027]"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_3c512a63","line":471,"in_reply_to":"9a8ffd7b_cff5decb","updated":"2015-12-01 18:59:59.000000000","message":"hmmm, apparently this is unused. Was a hangover from is_obsolete. Thanks.","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":475,"context_line":"        ts_other \u003d self.parse_on_disk_filename(other_filename)[\u0027timestamp\u0027]"},{"line_number":476,"context_line":"        return ts \u003c ts_other"},{"line_number":477,"context_line":""},{"line_number":478,"context_line":"    def _process_ondisk_files(self, exts, context, **kwargs):"},{"line_number":479,"context_line":"        \"\"\""},{"line_number":480,"context_line":"        Called by get_ondisk_files(). Should be over-ridden to implement"},{"line_number":481,"context_line":"        subclass specific handling of files."}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_afe65a14","line":478,"updated":"2015-12-01 01:13:08.000000000","message":"s/context/results/ to be consistent with the implementations in the subclass I guess?","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":475,"context_line":"        ts_other \u003d self.parse_on_disk_filename(other_filename)[\u0027timestamp\u0027]"},{"line_number":476,"context_line":"        return ts \u003c ts_other"},{"line_number":477,"context_line":""},{"line_number":478,"context_line":"    def _process_ondisk_files(self, exts, context, **kwargs):"},{"line_number":479,"context_line":"        \"\"\""},{"line_number":480,"context_line":"        Called by get_ondisk_files(). Should be over-ridden to implement"},{"line_number":481,"context_line":"        subclass specific handling of files."}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_fc8f22ca","line":478,"in_reply_to":"9a8ffd7b_afe65a14","updated":"2015-12-01 18:59:59.000000000","message":"Yes. To be consistent with its own docstring even!","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        into two: items newer than any in other_file_info and items older than"},{"line_number":503,"context_line":"        any in other_file_info."},{"line_number":504,"context_line":"        :param file_info_list: a list if file_info dicts"},{"line_number":505,"context_line":"        :param other_file_info_list: a list of file_info dicts"},{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_949fe6b3","line":505,"updated":"2015-12-01 01:13:08.000000000","message":"do we ever call with len(other_file_info_list) \u003e 1 ???","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        into two: items newer than any in other_file_info and items older than"},{"line_number":503,"context_line":"        any in other_file_info."},{"line_number":504,"context_line":"        :param file_info_list: a list if file_info dicts"},{"line_number":505,"context_line":"        :param other_file_info_list: a list of file_info dicts"},{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_551f215b","line":505,"in_reply_to":"9a8ffd7b_949fe6b3","updated":"2015-12-01 18:59:59.000000000","message":"No. I made the arg a list as a (horrible) convenience so that I could pass empty lists when no other_file exists, to save testing for existence at each call site. It turns out that with a little re-ordering of code I get the test for existence for free at almost all call sites.\n\n#willfix","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"},{"line_number":509,"context_line":"                            second returned list."},{"line_number":510,"context_line":"        :return: two lists of file_info dicts having newer and older items"},{"line_number":511,"context_line":"                 respectively."},{"line_number":512,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_4fbeced5","line":509,"updated":"2015-12-01 01:13:08.000000000","message":"gross\n\nok, after a little more time with it - it turns out this is sort of like a bisect algorithm.\n\nWith frags, you might have many items in the first list, with everything else there\u0027s only one.  Anything older (or eq except in the case of frags) goes in the left.  So at each call site it\u0027s pretty clear to see that split_right is normally False except when you want to deal with the special case of multiple frags at the same timestamp.\n\nMy guess is right now that _split_by_time is only useful when building fragsets, and otherwise it doesn\u0027t matter a whole lot which way equal timestamps go, and we might be able to drop split_right as kwarg.  For everything that\u0027s not a frag (including replicated .data) - I expect there exists a simpler wrapper around this method that would better expose the split_right\u003dFalse case by making some sort of deterministic decision about equal timestamps showing up when they should represent the exact same data and return (newer, obsolete) directly. Maybe.","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"},{"line_number":509,"context_line":"                            second returned list."},{"line_number":510,"context_line":"        :return: two lists of file_info dicts having newer and older items"},{"line_number":511,"context_line":"                 respectively."},{"line_number":512,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_30bce16b","line":509,"in_reply_to":"9a8ffd7b_4fbeced5","updated":"2015-12-01 18:59:59.000000000","message":"This method seems to have cost more review time than it should so i will re-work it to hopefully make it easier to grok. I\u0027m going to break this into two method to make the meaning of \"split_right\" clearer.\n\nYes, its similar outcome to a bisect (although obviously not a bisection algorithm which is why I didn\u0027t use bisect in the method name). I can\u0027t use bisect because it doesn\u0027t accept custom comparators/keys and I want to compare timestamps. But bisect gave me the notion of \"right\" as in \"split_right\".\n\nThe first list can have multiple items for files other than frags when cleanup may have previously failed, leaving stale files in the data dir. Also, in future we may expect multiple .meta files with fast-post.","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":556,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":557,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":558,"context_line":"            except (DiskFileError, ValueError):"},{"line_number":559,"context_line":"                # ignore unexpected files"},{"line_number":560,"context_line":"                pass"},{"line_number":561,"context_line":"        for ext in exts:"},{"line_number":562,"context_line":"            # For each extension sort files into reverse chronological order."}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_1916d455","line":559,"updated":"2015-12-01 01:13:08.000000000","message":"if it\u0027s unexpected we should log something so that we can learn to expect them if it happens.  If there *are* expected failures we should have a tighter condition so we don\u0027t pass or something we could otherwise learn from.\n\nI think handling errors like this and logging a warning is pretty tight:\n\nhttps://gist.github.com/clayg/12c314a7f95fa438e662\n\nI even had a chance to test it with a misconfig on my my saio where my policy-0 had changed from EC to replicated type and I didn\u0027t wipe my devices leading to a great warning in the logs (in the gist as an additional file)","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":556,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":557,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":558,"context_line":"            except (DiskFileError, ValueError):"},{"line_number":559,"context_line":"                # ignore unexpected files"},{"line_number":560,"context_line":"                pass"},{"line_number":561,"context_line":"        for ext in exts:"},{"line_number":562,"context_line":"            # For each extension sort files into reverse chronological order."}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_d3c9df7a","line":559,"in_reply_to":"9a8ffd7b_1916d455","updated":"2015-12-01 18:59:59.000000000","message":"OK, Paul mentioned this too - I had maintained existing behaviour (ignoring unexpected files) but I\u0027m happy to log them. Thanks for the diff!!","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":619,"context_line":"            # the file is old enough to be reclaimed"},{"line_number":620,"context_line":"            try:"},{"line_number":621,"context_line":"                timestamp \u003d file_.get(\u0027timestamp\u0027)"},{"line_number":622,"context_line":"            except AttributeError:"},{"line_number":623,"context_line":"                timestamp \u003d self.parse_on_disk_filename(file_)[\u0027timestamp\u0027]"},{"line_number":624,"context_line":"            return (time.time() - float(timestamp)) \u003e reclaim_age"},{"line_number":625,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_192a542d","line":622,"updated":"2015-12-01 01:13:08.000000000","message":"gross.  I think we\u0027ve just thrown away too much useful information too early:\n\nhttps://gist.github.com/clayg/039a19b9d1d320b01a5d","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":619,"context_line":"            # the file is old enough to be reclaimed"},{"line_number":620,"context_line":"            try:"},{"line_number":621,"context_line":"                timestamp \u003d file_.get(\u0027timestamp\u0027)"},{"line_number":622,"context_line":"            except AttributeError:"},{"line_number":623,"context_line":"                timestamp \u003d self.parse_on_disk_filename(file_)[\u0027timestamp\u0027]"},{"line_number":624,"context_line":"            return (time.time() - float(timestamp)) \u003e reclaim_age"},{"line_number":625,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_78f9dec7","line":622,"in_reply_to":"9a8ffd7b_192a542d","updated":"2015-12-01 18:59:59.000000000","message":"Done","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":2052,"context_line":"        # .data\u0027s, is obsolete"},{"line_number":2053,"context_line":"        for ext in exts.keys():"},{"line_number":2054,"context_line":"            exts[ext], older \u003d self._split_by_time("},{"line_number":2055,"context_line":"                exts[ext], exts[\u0027.data\u0027][:1], split_right\u003dTrue)"},{"line_number":2056,"context_line":"            results.setdefault(\u0027obsolete\u0027, []).extend(older)"},{"line_number":2057,"context_line":""},{"line_number":2058,"context_line":"        # set results"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_34087a28","line":2055,"updated":"2015-12-01 01:13:08.000000000","message":"should this be split_right\u003dTrue?  Keep equals?","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":2052,"context_line":"        # .data\u0027s, is obsolete"},{"line_number":2053,"context_line":"        for ext in exts.keys():"},{"line_number":2054,"context_line":"            exts[ext], older \u003d self._split_by_time("},{"line_number":2055,"context_line":"                exts[ext], exts[\u0027.data\u0027][:1], split_right\u003dTrue)"},{"line_number":2056,"context_line":"            results.setdefault(\u0027obsolete\u0027, []).extend(older)"},{"line_number":2057,"context_line":""},{"line_number":2058,"context_line":"        # set results"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_15dbd9ee","line":2055,"in_reply_to":"9a8ffd7b_34087a28","updated":"2015-12-01 18:59:59.000000000","message":"Yes otherwise we\u0027d throw away the most recent .data.\n\nBut, there\u0027s no reason to keep equal timestamps for other extensions.\n\n#willfix","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":2057,"context_line":""},{"line_number":2058,"context_line":"        # set results"},{"line_number":2059,"context_line":"        if exts[\u0027.data\u0027]:"},{"line_number":2060,"context_line":"            results[\u0027.data\u0027] \u003d exts[\u0027.data\u0027][0][\u0027filename\u0027]"},{"line_number":2061,"context_line":""},{"line_number":2062,"context_line":"    def _verify_on_disk_files(self, accepted_files, **kwargs):"},{"line_number":2063,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_1410d64d","line":2060,"updated":"2015-12-01 01:13:08.000000000","message":"... idk, doesn\u0027t seem to care to much if len(exts[\u003ckey\u003e]) \u003e 1","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":2057,"context_line":""},{"line_number":2058,"context_line":"        # set results"},{"line_number":2059,"context_line":"        if exts[\u0027.data\u0027]:"},{"line_number":2060,"context_line":"            results[\u0027.data\u0027] \u003d exts[\u0027.data\u0027][0][\u0027filename\u0027]"},{"line_number":2061,"context_line":""},{"line_number":2062,"context_line":"    def _verify_on_disk_files(self, accepted_files, **kwargs):"},{"line_number":2063,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_95bb4975","line":2060,"in_reply_to":"9a8ffd7b_1410d64d","updated":"2015-12-01 18:59:59.000000000","message":"Done","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":2398,"context_line":"        for frag_set in frag_sets.values():"},{"line_number":2399,"context_line":"            if frag_set \u003d\u003d durable_frag_set:"},{"line_number":2400,"context_line":"                continue"},{"line_number":2401,"context_line":"            results.setdefault(\u0027possible_reclaim\u0027, []).extend(frag_set)"},{"line_number":2402,"context_line":""},{"line_number":2403,"context_line":"    def _verify_on_disk_files(self, accepted_files, frag_index\u003dNone, **kwargs):"},{"line_number":2404,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_b1f0fcca","line":2401,"updated":"2015-12-01 01:13:08.000000000","message":"this is honestly quite manageable - kudos.","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":466,"context_line":"        subclass specific handling of files."},{"line_number":467,"context_line":""},{"line_number":468,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":469,"context_line":"        :param results: a dict to hold results"},{"line_number":470,"context_line":"        \"\"\""},{"line_number":471,"context_line":"        raise NotImplementedError"},{"line_number":472,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_5e4fdd33","line":469,"updated":"2015-12-02 23:25:31.000000000","message":"nit: I think the replicated and ec case mostly just return None - this method *updates* the passed in results - but the return value is ignored in get_ondisk_files","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":466,"context_line":"        subclass specific handling of files."},{"line_number":467,"context_line":""},{"line_number":468,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":469,"context_line":"        :param results: a dict to hold results"},{"line_number":470,"context_line":"        \"\"\""},{"line_number":471,"context_line":"        raise NotImplementedError"},{"line_number":472,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_6c33f762","line":469,"in_reply_to":"9a8ffd7b_5e4fdd33","updated":"2015-12-03 15:50:23.000000000","message":"Done - state that the dict gets updated.\n\nOr did you mean I should explicity have a :returns: None ?","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":615,"context_line":"            key \u003d info_key[:-5] + \u0027_file\u0027"},{"line_number":616,"context_line":"            results[key] \u003d join(datadir, info[\u0027filename\u0027]) if info else None"},{"line_number":617,"context_line":""},{"line_number":618,"context_line":"        return results"},{"line_number":619,"context_line":""},{"line_number":620,"context_line":"    def cleanup_ondisk_files(self, hsh_path, reclaim_age\u003dONE_WEEK, **kwargs):"},{"line_number":621,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_5bf60f5e","line":618,"updated":"2015-12-02 23:25:31.000000000","message":"it\u0027s even *better* now - amazing","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2090,"context_line":"        \"\"\""},{"line_number":2091,"context_line":"        # mimic legacy behavior - .meta is ignored when .ts is found"},{"line_number":2092,"context_line":"        if accepted_files.get(\u0027.ts\u0027):"},{"line_number":2093,"context_line":"            accepted_files.pop(\u0027.meta\u0027, None)"},{"line_number":2094,"context_line":""},{"line_number":2095,"context_line":"        data_file, meta_file, ts_file \u003d tuple("},{"line_number":2096,"context_line":"            [accepted_files.get(ext)"}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_91aabe84","line":2093,"updated":"2015-12-03 15:50:23.000000000","message":"this is no longer necessary - get_ondisk_files does the right thing","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2094,"context_line":""},{"line_number":2095,"context_line":"        data_file, meta_file, ts_file \u003d tuple("},{"line_number":2096,"context_line":"            [accepted_files.get(ext)"},{"line_number":2097,"context_line":"             for ext in (\u0027.data\u0027, \u0027.meta\u0027, \u0027.ts\u0027)])"},{"line_number":2098,"context_line":""},{"line_number":2099,"context_line":"        return ((data_file is None and meta_file is None and ts_file is None)"},{"line_number":2100,"context_line":"                or (ts_file is not None and data_file is None"}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_8f40b1a6","line":2097,"updated":"2015-12-02 23:25:31.000000000","message":"how is this even working right now?  accepted_files never has these keys anymore!?","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2094,"context_line":""},{"line_number":2095,"context_line":"        data_file, meta_file, ts_file \u003d tuple("},{"line_number":2096,"context_line":"            [accepted_files.get(ext)"},{"line_number":2097,"context_line":"             for ext in (\u0027.data\u0027, \u0027.meta\u0027, \u0027.ts\u0027)])"},{"line_number":2098,"context_line":""},{"line_number":2099,"context_line":"        return ((data_file is None and meta_file is None and ts_file is None)"},{"line_number":2100,"context_line":"                or (ts_file is not None and data_file is None"}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_40c7d795","line":2097,"in_reply_to":"9a8ffd7b_8f40b1a6","updated":"2015-12-03 15:50:23.000000000","message":"Done","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2238,"context_line":"        \"\"\""},{"line_number":2239,"context_line":"        The only difference between this method and the replication policy"},{"line_number":2240,"context_line":"        DiskFile method is passing in the frag_index kwarg to our manager\u0027s"},{"line_number":2241,"context_line":"        get_ondisk_files method, and post processing the file info."},{"line_number":2242,"context_line":"        \"\"\""},{"line_number":2243,"context_line":"        results \u003d self.manager.get_ondisk_files("},{"line_number":2244,"context_line":"            files, self._datadir, frag_index\u003dself._frag_index)"}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_7e4c2142","line":2241,"updated":"2015-12-02 23:25:31.000000000","message":"nit: well really the frag_index, and keeping the _ondisk_info attribute around - which could be bubbled up","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2238,"context_line":"        \"\"\""},{"line_number":2239,"context_line":"        The only difference between this method and the replication policy"},{"line_number":2240,"context_line":"        DiskFile method is passing in the frag_index kwarg to our manager\u0027s"},{"line_number":2241,"context_line":"        get_ondisk_files method, and post processing the file info."},{"line_number":2242,"context_line":"        \"\"\""},{"line_number":2243,"context_line":"        results \u003d self.manager.get_ondisk_files("},{"line_number":2244,"context_line":"            files, self._datadir, frag_index\u003dself._frag_index)"}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_0e20ebf6","line":2241,"in_reply_to":"9a8ffd7b_7e4c2142","updated":"2015-12-03 15:50:23.000000000","message":"Done","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2394,"context_line":"            timestamp \u003d frag_set[0][\u0027timestamp\u0027]"},{"line_number":2395,"context_line":"            frag_sets[timestamp] \u003d frag_set"},{"line_number":2396,"context_line":"            if durable_info and durable_info[\u0027timestamp\u0027] \u003d\u003d timestamp:"},{"line_number":2397,"context_line":"                durable_frag_set \u003d frag_set"},{"line_number":2398,"context_line":""},{"line_number":2399,"context_line":"        # Select a single chosen frag from the chosen frag_set, by either"},{"line_number":2400,"context_line":"        # matching against a specified frag_index or taking the highest index."}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_84cbb852","line":2397,"updated":"2015-12-02 23:25:31.000000000","message":"yes!  comment was great.","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2404,"context_line":"                # search the frag set to find the exact frag_index"},{"line_number":2405,"context_line":"                for info in durable_frag_set:"},{"line_number":2406,"context_line":"                    if info[\u0027frag_index\u0027] \u003d\u003d frag_index:"},{"line_number":2407,"context_line":"                        chosen_frag \u003d info"},{"line_number":2408,"context_line":"            else:"},{"line_number":2409,"context_line":"                chosen_frag \u003d durable_frag_set[-1]"},{"line_number":2410,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_c431c05a","line":2407,"updated":"2015-12-02 23:25:31.000000000","message":"nit: break","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2404,"context_line":"                # search the frag set to find the exact frag_index"},{"line_number":2405,"context_line":"                for info in durable_frag_set:"},{"line_number":2406,"context_line":"                    if info[\u0027frag_index\u0027] \u003d\u003d frag_index:"},{"line_number":2407,"context_line":"                        chosen_frag \u003d info"},{"line_number":2408,"context_line":"            else:"},{"line_number":2409,"context_line":"                chosen_frag \u003d durable_frag_set[-1]"},{"line_number":2410,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_ceed0390","line":2407,"in_reply_to":"9a8ffd7b_c431c05a","updated":"2015-12-03 15:50:23.000000000","message":"Done","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2409,"context_line":"                chosen_frag \u003d durable_frag_set[-1]"},{"line_number":2410,"context_line":""},{"line_number":2411,"context_line":"        # If we successfully found a frag then set results"},{"line_number":2412,"context_line":"        if chosen_frag:"},{"line_number":2413,"context_line":"            results[\u0027data_info\u0027] \u003d chosen_frag"},{"line_number":2414,"context_line":"            results[\u0027durable_frag_set\u0027] \u003d durable_frag_set"},{"line_number":2415,"context_line":"        results[\u0027frag_sets\u0027] \u003d frag_sets"}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_449070ed","line":2412,"updated":"2015-12-02 23:25:31.000000000","message":"nit: this *implies* durable right?  You *could* have durable, but not find a frag with the specified index - but the only way you have a choosen frag is if you\u0027re durable - maybe it can be rolled under he above condition?","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2409,"context_line":"                chosen_frag \u003d durable_frag_set[-1]"},{"line_number":2410,"context_line":""},{"line_number":2411,"context_line":"        # If we successfully found a frag then set results"},{"line_number":2412,"context_line":"        if chosen_frag:"},{"line_number":2413,"context_line":"            results[\u0027data_info\u0027] \u003d chosen_frag"},{"line_number":2414,"context_line":"            results[\u0027durable_frag_set\u0027] \u003d durable_frag_set"},{"line_number":2415,"context_line":"        results[\u0027frag_sets\u0027] \u003d frag_sets"}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_f178e291","line":2412,"in_reply_to":"9a8ffd7b_449070ed","updated":"2015-12-03 15:50:23.000000000","message":"yes. but future-me already knows that for optimistic non-durable GET responses (https://review.openstack.org/#/c/215276) having a chosen_frag does not necessarily mean that you have a durable frag set.\n\nSo if we can live with this nit then it will make a future change simpler.","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ca81e28911dc41a75037e0678b05921879121df6","unresolved":false,"context_lines":[{"line_number":2417,"context_line":"        # Mark any isolated .durable as obsolete"},{"line_number":2418,"context_line":"        if exts[\u0027.durable\u0027] and not durable_frag_set:"},{"line_number":2419,"context_line":"            results.setdefault(\u0027obsolete\u0027, []).extend(exts[\u0027.durable\u0027])"},{"line_number":2420,"context_line":"            exts.pop(\u0027.durable\u0027)"},{"line_number":2421,"context_line":""},{"line_number":2422,"context_line":"        # Fragments *may* be ready for reclaim, unless they are durable or"},{"line_number":2423,"context_line":"        # at the timestamp we have just chosen for constructing the diskfile."}],"source_content_type":"text/x-python","patch_set":6,"id":"9a8ffd7b_8f73d17f","line":2420,"updated":"2015-12-02 23:25:31.000000000","message":"I guess purge currently leaves .durable files isolated - so this clean up is probably good.","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7fda8cf5e6f7c3b818ce307e4efe277c129440c7","unresolved":false,"context_lines":[{"line_number":2435,"context_line":"        :param frag_index: specifies a specific fragment index .data file"},{"line_number":2436,"context_line":"        :returns: True if the file combination is compliant, False otherwise"},{"line_number":2437,"context_line":"        \"\"\""},{"line_number":2438,"context_line":"        if not accepted_files.get(\u0027.data\u0027):"},{"line_number":2439,"context_line":"            # We may find only a .meta, which doesn\u0027t mean the on disk"},{"line_number":2440,"context_line":"            # contract is broken. So we clear it to comply with"},{"line_number":2441,"context_line":"            # superclass assertions."}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_319c2a69","line":2438,"updated":"2015-12-03 15:50:23.000000000","message":"this is no longer necessary - get_ondisk_files does the right thing","commit_id":"8791b00643d496ed778f88c4f9648ac4987b6c29"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":494,"context_line":"        len(original list). The second list contains the remaining items of the"},{"line_number":495,"context_line":"        original list, in their original order. The index, N, at which the"},{"line_number":496,"context_line":"        original list is split, is the index of the first item in the list that"},{"line_number":497,"context_line":"        does not satisfy the given condition."},{"line_number":498,"context_line":""},{"line_number":499,"context_line":"        :param original_list: the list to be split."},{"line_number":500,"context_line":"        :param condition: a single argument function that will be used to test"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_0b29f59a","line":497,"updated":"2015-12-14 09:33:58.000000000","message":"Nits) I don\u0027t think that\u0027s not so a big matter but I like to add a note that the give original list should be ordered by splittable view by given condition.\n\ni.e. if the original list is ordered by *name* and the given condition is corresponding to *timestamp*, it may be able to split somewhere in the list but the later list *may* includes items which fit the condition.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":494,"context_line":"        len(original list). The second list contains the remaining items of the"},{"line_number":495,"context_line":"        original list, in their original order. The index, N, at which the"},{"line_number":496,"context_line":"        original list is split, is the index of the first item in the list that"},{"line_number":497,"context_line":"        does not satisfy the given condition."},{"line_number":498,"context_line":""},{"line_number":499,"context_line":"        :param original_list: the list to be split."},{"line_number":500,"context_line":"        :param condition: a single argument function that will be used to test"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_2b11f152","line":497,"in_reply_to":"7a740942_0b29f59a","updated":"2015-12-15 15:36:50.000000000","message":"Done","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":576,"context_line":"            # Categorize files by extension"},{"line_number":577,"context_line":"            try:"},{"line_number":578,"context_line":"                file_info \u003d self.parse_on_disk_filename(afile)"},{"line_number":579,"context_line":"                file_info[\u0027filename\u0027] \u003d afile"},{"line_number":580,"context_line":"                exts[file_info[\u0027ext\u0027]].append(file_info)"},{"line_number":581,"context_line":"            except DiskFileError as e:"},{"line_number":582,"context_line":"                self.logger.warning(\u0027Unexpected file %s: %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_0b4815b2","line":579,"updated":"2015-12-14 09:33:58.000000000","message":"memo) filename is like ts.ext, file info is like {\u0027timestamp\u0027: Timestamp, \u0027ext\u0027: str(ext)}","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":590,"context_line":"        results \u003d {}"},{"line_number":591,"context_line":""},{"line_number":592,"context_line":"        # non-tombstones older than or equal to latest tombstone are obsolete"},{"line_number":593,"context_line":"        if exts.get(\u0027.ts\u0027):"},{"line_number":594,"context_line":"            for ext in filter(lambda ext: ext !\u003d \u0027.ts\u0027, exts.keys()):"},{"line_number":595,"context_line":"                exts[ext], older \u003d self._split_gt_timestamp("},{"line_number":596,"context_line":"                    exts[ext], exts[\u0027.ts\u0027][0][\u0027timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_7d0de0e6","line":593,"updated":"2015-12-14 09:33:58.000000000","message":"Nits) Should we change this exts[\u0027.ts\u0027]?\n\nThat is because this looks to have an inconsistency with other code in this file. (e.g. L2078)\n\nThe inconsistency is \"exts[ext]\" vs \"exts.get(ext)\" for if conditions.\n\nI think exts is a defaultdict and if the key doesn\u0027t exist, both ways will be no matter because it will be an empty list or None (i.e. False condition).\n\nEither is fine to me, tho., IMO, it is probably good to change L2078 to use \"exts.get()\" to reduce to make extra empty lists.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":590,"context_line":"        results \u003d {}"},{"line_number":591,"context_line":""},{"line_number":592,"context_line":"        # non-tombstones older than or equal to latest tombstone are obsolete"},{"line_number":593,"context_line":"        if exts.get(\u0027.ts\u0027):"},{"line_number":594,"context_line":"            for ext in filter(lambda ext: ext !\u003d \u0027.ts\u0027, exts.keys()):"},{"line_number":595,"context_line":"                exts[ext], older \u003d self._split_gt_timestamp("},{"line_number":596,"context_line":"                    exts[ext], exts[\u0027.ts\u0027][0][\u0027timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_cb4e2dc1","line":593,"in_reply_to":"7a740942_7d0de0e6","updated":"2015-12-15 15:36:50.000000000","message":"Done. There were some other places I will change to exts.get(key) instead of exts[key]","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":602,"context_line":"            exts[ext] \u003d exts[ext][:1]"},{"line_number":603,"context_line":""},{"line_number":604,"context_line":"        # delegate to subclass handler"},{"line_number":605,"context_line":"        self._process_ondisk_files(exts, results, **kwargs)"},{"line_number":606,"context_line":""},{"line_number":607,"context_line":"        # set final choice of files"},{"line_number":608,"context_line":"        if exts[\u0027.ts\u0027]:"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_1d14dce6","line":605,"updated":"2015-12-14 09:33:58.000000000","message":"memo) _process_ondisk_files *should* remove older timestamp items from exts (e.g. exts[\u0027.ts\u0027], exts[\u0027.meta\u0027])","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":605,"context_line":"        self._process_ondisk_files(exts, results, **kwargs)"},{"line_number":606,"context_line":""},{"line_number":607,"context_line":"        # set final choice of files"},{"line_number":608,"context_line":"        if exts[\u0027.ts\u0027]:"},{"line_number":609,"context_line":"            results[\u0027ts_info\u0027] \u003d exts[\u0027.ts\u0027][0]"},{"line_number":610,"context_line":"        if \u0027data_info\u0027 in results and exts[\u0027.meta\u0027]:"},{"line_number":611,"context_line":"            # only report a meta file if there is a data file"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_9383db02","line":608,"updated":"2015-12-14 09:33:58.000000000","message":"In-consistency with L593.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":605,"context_line":"        self._process_ondisk_files(exts, results, **kwargs)"},{"line_number":606,"context_line":""},{"line_number":607,"context_line":"        # set final choice of files"},{"line_number":608,"context_line":"        if exts[\u0027.ts\u0027]:"},{"line_number":609,"context_line":"            results[\u0027ts_info\u0027] \u003d exts[\u0027.ts\u0027][0]"},{"line_number":610,"context_line":"        if \u0027data_info\u0027 in results and exts[\u0027.meta\u0027]:"},{"line_number":611,"context_line":"            # only report a meta file if there is a data file"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_66b970e9","line":608,"in_reply_to":"7a740942_9383db02","updated":"2015-12-15 15:36:50.000000000","message":"Done","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":609,"context_line":"            results[\u0027ts_info\u0027] \u003d exts[\u0027.ts\u0027][0]"},{"line_number":610,"context_line":"        if \u0027data_info\u0027 in results and exts[\u0027.meta\u0027]:"},{"line_number":611,"context_line":"            # only report a meta file if there is a data file"},{"line_number":612,"context_line":"            results[\u0027meta_info\u0027] \u003d exts[\u0027.meta\u0027][0]"},{"line_number":613,"context_line":""},{"line_number":614,"context_line":"        # set ts_file, data_file and meta_file with path to chosen file or None"},{"line_number":615,"context_line":"        for info_key in (\u0027data_info\u0027, \u0027meta_info\u0027, \u0027ts_info\u0027):"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_4cb2b04d","line":612,"updated":"2015-12-14 09:33:58.000000000","message":"memo) This block above means we have newer objects than \u0027.data\u0027 (in replicated) or \u0027.data, .durable\u0027 (in EC)","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":2075,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":2076,"context_line":"        :param results: a dict that may be updated with results"},{"line_number":2077,"context_line":"        \"\"\""},{"line_number":2078,"context_line":"        if exts[\u0027.data\u0027]:"},{"line_number":2079,"context_line":"            for ext in exts.keys():"},{"line_number":2080,"context_line":"                if ext \u003d\u003d \u0027.data\u0027:"},{"line_number":2081,"context_line":"                    # older .data\u0027s are obsolete"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_dd3174a3","line":2078,"updated":"2015-12-14 09:33:58.000000000","message":"In-consistency, here (See L593 comment)","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":2075,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":2076,"context_line":"        :param results: a dict that may be updated with results"},{"line_number":2077,"context_line":"        \"\"\""},{"line_number":2078,"context_line":"        if exts[\u0027.data\u0027]:"},{"line_number":2079,"context_line":"            for ext in exts.keys():"},{"line_number":2080,"context_line":"                if ext \u003d\u003d \u0027.data\u0027:"},{"line_number":2081,"context_line":"                    # older .data\u0027s are obsolete"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_86a1fcab","line":2078,"in_reply_to":"7a740942_dd3174a3","updated":"2015-12-15 15:36:50.000000000","message":"Done","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":2084,"context_line":"                else:"},{"line_number":2085,"context_line":"                    # other files at same or older timestamp as most recent"},{"line_number":2086,"context_line":"                    # data are obsolete"},{"line_number":2087,"context_line":"                    exts[ext], older \u003d self._split_gt_timestamp("},{"line_number":2088,"context_line":"                        exts[ext], exts[\u0027.data\u0027][0][\u0027timestamp\u0027])"},{"line_number":2089,"context_line":"                results.setdefault(\u0027obsolete\u0027, []).extend(older)"},{"line_number":2090,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_7de640b1","line":2087,"updated":"2015-12-14 09:33:58.000000000","message":"Nits) s/older/same_or_older/, strictly.\n\nPlus... it might be only me but it looks odd a bit that \"_split_*gte*_timestamp\" makes  a tuple of list consists of ([smaller than or equals to], [greater than]) even though gte means \"greater than or equals\". \n\nOn the other hand \"_split_*gt*_timestamp\" will make a tuple ([smaller than], [greater than and equals to])  \n\nI have no good idea to improve for this yet but just note here.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":2084,"context_line":"                else:"},{"line_number":2085,"context_line":"                    # other files at same or older timestamp as most recent"},{"line_number":2086,"context_line":"                    # data are obsolete"},{"line_number":2087,"context_line":"                    exts[ext], older \u003d self._split_gt_timestamp("},{"line_number":2088,"context_line":"                        exts[ext], exts[\u0027.data\u0027][0][\u0027timestamp\u0027])"},{"line_number":2089,"context_line":"                results.setdefault(\u0027obsolete\u0027, []).extend(older)"},{"line_number":2090,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_267a8825","line":2087,"in_reply_to":"7a740942_7de640b1","updated":"2015-12-15 15:36:50.000000000","message":"Done, changed variable name.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c8556854d8d9c00f75c70c7df1be8d148f55fc57","unresolved":false,"context_lines":[{"line_number":2084,"context_line":"                else:"},{"line_number":2085,"context_line":"                    # other files at same or older timestamp as most recent"},{"line_number":2086,"context_line":"                    # data are obsolete"},{"line_number":2087,"context_line":"                    exts[ext], older \u003d self._split_gt_timestamp("},{"line_number":2088,"context_line":"                        exts[ext], exts[\u0027.data\u0027][0][\u0027timestamp\u0027])"},{"line_number":2089,"context_line":"                results.setdefault(\u0027obsolete\u0027, []).extend(older)"},{"line_number":2090,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_6fbb970f","line":2087,"in_reply_to":"7a740942_7de640b1","updated":"2015-12-15 00:41:41.000000000","message":"Sorry, I revert my comment since \"Plus\". That is because the list is reverse ordered. \n\ni.e.\n- _split_gt_timestamp -\u003e ([greater than], [equals to or smaller than]\n- _split_gte_timestamp -\u003e ([greater than or equals to], [smaller])\n\nSo the function name fits actual behavior. I was misunderstanding that.\n\n(perhaps, it might be good \"lt\" and \"lte\" to cut off the tail less than exts[\u0027.data\u0027][0], tho)","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":2359,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2360,"context_line":"        \"\"\""},{"line_number":2361,"context_line":"        durable_info \u003d None"},{"line_number":2362,"context_line":"        if exts[\u0027.durable\u0027]:"},{"line_number":2363,"context_line":"            durable_info \u003d exts[\u0027.durable\u0027][0]"},{"line_number":2364,"context_line":"            # Mark everything older than most recent .durable as obsolete"},{"line_number":2365,"context_line":"            # and remove from the exts dict."}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_9d368c4f","line":2362,"updated":"2015-12-14 09:33:58.000000000","message":"Same as L2078","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":2359,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2360,"context_line":"        \"\"\""},{"line_number":2361,"context_line":"        durable_info \u003d None"},{"line_number":2362,"context_line":"        if exts[\u0027.durable\u0027]:"},{"line_number":2363,"context_line":"            durable_info \u003d exts[\u0027.durable\u0027][0]"},{"line_number":2364,"context_line":"            # Mark everything older than most recent .durable as obsolete"},{"line_number":2365,"context_line":"            # and remove from the exts dict."}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_6603b074","line":2362,"in_reply_to":"7a740942_9d368c4f","updated":"2015-12-15 15:36:50.000000000","message":"Done","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":2400,"context_line":"                chosen_frag \u003d durable_frag_set[-1]"},{"line_number":2401,"context_line":""},{"line_number":2402,"context_line":"        # If we successfully found a frag then set results"},{"line_number":2403,"context_line":"        if chosen_frag:"},{"line_number":2404,"context_line":"            results[\u0027data_info\u0027] \u003d chosen_frag"},{"line_number":2405,"context_line":"            results[\u0027durable_frag_set\u0027] \u003d durable_frag_set"},{"line_number":2406,"context_line":"        results[\u0027frag_sets\u0027] \u003d frag_sets"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_d3796394","line":2403,"updated":"2015-12-14 09:33:58.000000000","message":"memo) if frag_index specified and not found, chosen_frag still be None. That will make no \u0027data_info\u0027.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":2415,"context_line":"        for frag_set in frag_sets.values():"},{"line_number":2416,"context_line":"            if frag_set \u003d\u003d durable_frag_set:"},{"line_number":2417,"context_line":"                continue"},{"line_number":2418,"context_line":"            results.setdefault(\u0027possible_reclaim\u0027, []).extend(frag_set)"},{"line_number":2419,"context_line":""},{"line_number":2420,"context_line":"    def _verify_ondisk_files(self, results, frag_index\u003dNone, **kwargs):"},{"line_number":2421,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_d30ac3d1","line":2418,"updated":"2015-12-14 09:33:58.000000000","message":"Nits) possible_reclaim will include an in-progress-writing object. (i.e. .data landed but .durable not yet) but it\u0027s not reclaimable hopefully. I hope no race will happen even if the reclaim_age is too small.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"d3cb969be73128142b009274196b017f930b95e1","unresolved":false,"context_lines":[{"line_number":2415,"context_line":"        for frag_set in frag_sets.values():"},{"line_number":2416,"context_line":"            if frag_set \u003d\u003d durable_frag_set:"},{"line_number":2417,"context_line":"                continue"},{"line_number":2418,"context_line":"            results.setdefault(\u0027possible_reclaim\u0027, []).extend(frag_set)"},{"line_number":2419,"context_line":""},{"line_number":2420,"context_line":"    def _verify_ondisk_files(self, results, frag_index\u003dNone, **kwargs):"},{"line_number":2421,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_01cb7f1f","line":2418,"in_reply_to":"5a710552_c62f04a0","updated":"2015-12-16 08:39:46.000000000","message":"Thanks for writing down more concretely. That fits to my idea and it might be good to add a constraint for that, imo. :)","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":2415,"context_line":"        for frag_set in frag_sets.values():"},{"line_number":2416,"context_line":"            if frag_set \u003d\u003d durable_frag_set:"},{"line_number":2417,"context_line":"                continue"},{"line_number":2418,"context_line":"            results.setdefault(\u0027possible_reclaim\u0027, []).extend(frag_set)"},{"line_number":2419,"context_line":""},{"line_number":2420,"context_line":"    def _verify_ondisk_files(self, results, frag_index\u003dNone, **kwargs):"},{"line_number":2421,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_c62f04a0","line":2418,"in_reply_to":"7a740942_d30ac3d1","updated":"2015-12-15 15:36:50.000000000","message":"Interesting :) If reclaim_age is configured so small (e.g. O(secs)) then I\u0027m not sure how we would avoid this happening. We need some logic that will remove non-durable .data *eventually*, and if reclaim_age is set so small that it is less than the latency between .data and .durable being written then that would be a configuration mistake. Perhaps we could enforce a minimum reclaim_age? If so , could that be a follow-up patch?","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"d73ca3c9a13138cae47a01de1c16bb72ab056986","unresolved":false,"context_lines":[{"line_number":1565,"context_line":"        Provides the timestamp of the newest data file found in the object"},{"line_number":1566,"context_line":"        directory."},{"line_number":1567,"context_line":""},{"line_number":1568,"context_line":"        :return: A Timestamp instance, or None if no data file was found."},{"line_number":1569,"context_line":"        :raises DiskFileNotOpen: if the open() method has not been previously"},{"line_number":1570,"context_line":"                                 called on this instance."},{"line_number":1571,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5a710552_012ffff6","line":1568,"updated":"2015-12-16 08:55:26.000000000","message":"I see, good to hear.","commit_id":"60b2e02905d57f55169e506f4874b2334a1a68a5"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":786,"context_line":"        self.assertEqual(hash(ts_0), hash(ts_0_also))"},{"line_number":787,"context_line":"        d \u003d {ts_0: \u0027whatever\u0027}"},{"line_number":788,"context_line":"        self.assertIn(ts_0, d)  # sanity"},{"line_number":789,"context_line":"        self.assertIn(ts_0_also, d)"},{"line_number":790,"context_line":""},{"line_number":791,"context_line":""},{"line_number":792,"context_line":"class TestUtils(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_d48c8e81","line":789,"updated":"2015-12-01 01:13:08.000000000","message":"nice.","commit_id":"3649d9e6b2b386ddfe0d460fbd06331e9cd20a31"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":593,"context_line":"                df_mgr \u003d self.df_router[policy]"},{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_ed4400b8","line":596,"updated":"2015-11-10 22:34:51.000000000","message":"i don\u0027t totally get this, can you explain why the change from equal to subset?","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b57dedcee67e8e53974ab4d7e889a4e0ef4c491f","unresolved":false,"context_lines":[{"line_number":593,"context_line":"                df_mgr \u003d self.df_router[policy]"},{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_3a68ee86","line":596,"in_reply_to":"9a8ffd7b_8971d1e2","updated":"2015-11-26 16:28:09.000000000","message":"specifically, in this case the results dict has key \u0027obsolete\u0027 which is an empty list - its get initialised during get_ondisk_files.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":593,"context_line":"                df_mgr \u003d self.df_router[policy]"},{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_8971d1e2","line":596,"in_reply_to":"da85f559_ed4400b8","updated":"2015-11-25 19:18:13.000000000","message":"Its because the results dict coming back from get_ondisk_files can now contain keys other than those for the file set (.data, .ts etc) which we don\u0027t care about for this test.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":3203,"context_line":"        with df.open():"},{"line_number":3204,"context_line":"            self.assertEqual(df.durable_timestamp, ts_1.internal)"},{"line_number":3205,"context_line":"        ts_2 \u003d self.ts()"},{"line_number":3206,"context_line":"        df.write_metadata({\u0027X-Timestamp\u0027: ts_2.internal})"},{"line_number":3207,"context_line":"        with df.open():"},{"line_number":3208,"context_line":"            self.assertEqual(df.durable_timestamp, ts_1.internal)"},{"line_number":3209,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_403f1387","line":3206,"updated":"2015-11-10 22:34:51.000000000","message":"what exactly are we testing here? just that a newer ts w/o a durable doesn\u0027t open the wrong .data file?","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":3203,"context_line":"        with df.open():"},{"line_number":3204,"context_line":"            self.assertEqual(df.durable_timestamp, ts_1.internal)"},{"line_number":3205,"context_line":"        ts_2 \u003d self.ts()"},{"line_number":3206,"context_line":"        df.write_metadata({\u0027X-Timestamp\u0027: ts_2.internal})"},{"line_number":3207,"context_line":"        with df.open():"},{"line_number":3208,"context_line":"            self.assertEqual(df.durable_timestamp, ts_1.internal)"},{"line_number":3209,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_2960e500","line":3206,"in_reply_to":"da85f559_403f1387","updated":"2015-11-25 19:18:13.000000000","message":"It should be checking that writing metadata at a newer timestamp does NOT change the durable timestamp.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"558e75444f5a002b946575fd61f89636a42a9bad","unresolved":false,"context_lines":[{"line_number":3950,"context_line":"        df \u003d self._get_open_disk_file(ts\u003dts_1.internal, frag_index\u003d2)"},{"line_number":3951,"context_line":"        self.assertEqual(df.fragments, {ts_1: [0, 2]})"},{"line_number":3952,"context_line":""},{"line_number":3953,"context_line":"        # add a newer datafile for frag index 3"},{"line_number":3954,"context_line":"        ts_2 \u003d self.ts()"},{"line_number":3955,"context_line":"        policy \u003d POLICIES.default"},{"line_number":3956,"context_line":"        df_mgr \u003d self.df_router[policy]"}],"source_content_type":"text/x-python","patch_set":3,"id":"da85f559_c3478dba","line":3953,"updated":"2015-11-10 22:34:51.000000000","message":"could you add an optional commit arg to _get_open_disk_file() and just make one simple call here to create frag 3?  Actually I\u0027ll try it since my brain seems to be stalled from lack of use anyways...","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b57dedcee67e8e53974ab4d7e889a4e0ef4c491f","unresolved":false,"context_lines":[{"line_number":3950,"context_line":"        df \u003d self._get_open_disk_file(ts\u003dts_1.internal, frag_index\u003d2)"},{"line_number":3951,"context_line":"        self.assertEqual(df.fragments, {ts_1: [0, 2]})"},{"line_number":3952,"context_line":""},{"line_number":3953,"context_line":"        # add a newer datafile for frag index 3"},{"line_number":3954,"context_line":"        ts_2 \u003d self.ts()"},{"line_number":3955,"context_line":"        policy \u003d POLICIES.default"},{"line_number":3956,"context_line":"        df_mgr \u003d self.df_router[policy]"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_ba293e39","line":3953,"in_reply_to":"9a8ffd7b_693cade2","updated":"2015-11-26 16:28:09.000000000","message":"done - applied Paul\u0027s diff","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"834a7f0037a1b06fe02cda745284e714ac549818","unresolved":false,"context_lines":[{"line_number":3950,"context_line":"        df \u003d self._get_open_disk_file(ts\u003dts_1.internal, frag_index\u003d2)"},{"line_number":3951,"context_line":"        self.assertEqual(df.fragments, {ts_1: [0, 2]})"},{"line_number":3952,"context_line":""},{"line_number":3953,"context_line":"        # add a newer datafile for frag index 3"},{"line_number":3954,"context_line":"        ts_2 \u003d self.ts()"},{"line_number":3955,"context_line":"        policy \u003d POLICIES.default"},{"line_number":3956,"context_line":"        df_mgr \u003d self.df_router[policy]"}],"source_content_type":"text/x-python","patch_set":3,"id":"9a8ffd7b_693cade2","line":3953,"in_reply_to":"da85f559_c3478dba","updated":"2015-11-25 19:18:13.000000000","message":"great idea, i thought i had done that somewhere else actually.","commit_id":"c9c9a2cf31207d5b58e4397ce71757346044de58"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f78a5d71e68a04dbcf16f411d436e175d9501419","unresolved":false,"context_lines":[{"line_number":1741,"context_line":"                    ts1.internal + \u0027#2.data\u0027,"},{"line_number":1742,"context_line":"                    ts1.internal + \u0027.durable\u0027,"},{"line_number":1743,"context_line":"                    ts2.internal + \u0027.meta\u0027],"},{"line_number":1744,"context_line":"                # this one is wrong frag index"},{"line_number":1745,"context_line":"                \u00279aaaa92d072897b136b3fc06595b0456\u0027: ["},{"line_number":1746,"context_line":"                    ts1.internal + \u0027#7.data\u0027,"},{"line_number":1747,"context_line":"                    ts1.internal + \u0027.durable\u0027],"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_f4951288","side":"PARENT","line":1744,"updated":"2015-12-01 01:13:08.000000000","message":"I kind of expected to see another test like this to hit a bunch of the multiple frag scenarios...","commit_id":"ea3a0d38c1b9335f602d89b215983384eca85565"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8aaa4b4dddba57dd2ffcf7298a8ed3771473bb27","unresolved":false,"context_lines":[{"line_number":1741,"context_line":"                    ts1.internal + \u0027#2.data\u0027,"},{"line_number":1742,"context_line":"                    ts1.internal + \u0027.durable\u0027,"},{"line_number":1743,"context_line":"                    ts2.internal + \u0027.meta\u0027],"},{"line_number":1744,"context_line":"                # this one is wrong frag index"},{"line_number":1745,"context_line":"                \u00279aaaa92d072897b136b3fc06595b0456\u0027: ["},{"line_number":1746,"context_line":"                    ts1.internal + \u0027#7.data\u0027,"},{"line_number":1747,"context_line":"                    ts1.internal + \u0027.durable\u0027],"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_58673ab7","side":"PARENT","line":1744,"in_reply_to":"9a8ffd7b_f4951288","updated":"2015-12-01 18:59:59.000000000","message":"not sure what you have in mind? - multiple frags are tested below in test_yield_hashes_filters_frag_index","commit_id":"ea3a0d38c1b9335f602d89b215983384eca85565"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"},{"line_number":600,"context_line":"                actual \u003d df._get_ondisk_files(files)"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_366bf5b8","line":597,"updated":"2015-12-14 09:33:58.000000000","message":"assertDictContainsSubset looks to be deprecated since py 3.2[1] so I don\u0027t think we should use it because we will get py3.4 voting gate in near future.\n\n[1] https://docs.python.org/2/library/unittest.html#unittest.TestCase.assertDictContainsSubset","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"d3cb969be73128142b009274196b017f930b95e1","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"},{"line_number":600,"context_line":"                actual \u003d df._get_ondisk_files(files)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_b2bd1a6a","line":597,"in_reply_to":"5a710552_1aab208b","updated":"2015-12-16 08:39:46.000000000","message":"Never mind :/","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                datadir \u003d os.path.join(\u0027/srv/node/sdb1/\u0027,"},{"line_number":595,"context_line":"                                       diskfile.get_data_dir(policy))"},{"line_number":596,"context_line":"                actual \u003d df_mgr.get_ondisk_files(files, datadir)"},{"line_number":597,"context_line":"                self.assertDictContainsSubset(expected, actual)"},{"line_number":598,"context_line":"                # check diskfile under the hood"},{"line_number":599,"context_line":"                df \u003d self._get_diskfile(policy, frag_index\u003dfrag_index)"},{"line_number":600,"context_line":"                actual \u003d df._get_ondisk_files(files)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_1aab208b","line":597,"in_reply_to":"7a740942_366bf5b8","updated":"2015-12-15 15:36:50.000000000","message":"WHAAAT!! we only got it to use it since we dropped py2.6 !! :(\n\ngood spot, will fix.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":1065,"context_line":"        # construct every possible invalid combination of results"},{"line_number":1066,"context_line":"        vals \u003d (None, \u0027a_file\u0027)"},{"line_number":1067,"context_line":"        for ts_file, data_file, meta_file in ["},{"line_number":1068,"context_line":"                (a, b, c) for a in vals for b in vals for c in vals]:"},{"line_number":1069,"context_line":"            scenario \u003d {"},{"line_number":1070,"context_line":"                \u0027ts_file\u0027: ts_file,"},{"line_number":1071,"context_line":"                \u0027data_file\u0027: data_file,"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_3628d599","line":1068,"updated":"2015-12-14 09:33:58.000000000","message":"Awesome, I didn\u0027t know the expression for making all combinations.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":1514,"context_line":"            if scenario in ok_scenarios:"},{"line_number":1515,"context_line":"                continue"},{"line_number":1516,"context_line":"            self.assertFalse(mgr._verify_ondisk_files(scenario),"},{"line_number":1517,"context_line":"                             \u0027Unexpected result for scenario %s\u0027 % scenario)"},{"line_number":1518,"context_line":""},{"line_number":1519,"context_line":"    def test_parse_on_disk_filename(self):"},{"line_number":1520,"context_line":"        mgr \u003d self.df_router[POLICIES.default]"}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_ec303c58","line":1517,"updated":"2015-12-14 09:33:58.000000000","message":"Looks good.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1fa13e11a24a4c379008c76d9bf111600c231340","unresolved":false,"context_lines":[{"line_number":3278,"context_line":""},{"line_number":3279,"context_line":"    def test_durable_timestamp_not_open(self):"},{"line_number":3280,"context_line":"        df \u003d self._simple_get_diskfile()"},{"line_number":3281,"context_line":"        self.assertIsNone(df.durable_timestamp)"},{"line_number":3282,"context_line":""},{"line_number":3283,"context_line":"    def test_error_in_hash_cleanup_listdir(self):"},{"line_number":3284,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"7a740942_e2e283d4","line":3281,"updated":"2015-12-14 09:33:58.000000000","message":"Not sure for primary usage of durable_timestamp but if durable_timestamp is not available when the diskfile is not open yet, I like to make this either of following 2 options.\n\n 1. open the disk file (like ring reload)\n 2. raise DiskFileNotOpen()\n\nI like second option, tho.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"d3cb969be73128142b009274196b017f930b95e1","unresolved":false,"context_lines":[{"line_number":3278,"context_line":""},{"line_number":3279,"context_line":"    def test_durable_timestamp_not_open(self):"},{"line_number":3280,"context_line":"        df \u003d self._simple_get_diskfile()"},{"line_number":3281,"context_line":"        self.assertIsNone(df.durable_timestamp)"},{"line_number":3282,"context_line":""},{"line_number":3283,"context_line":"    def test_error_in_hash_cleanup_listdir(self):"},{"line_number":3284,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_92a33e4f","line":3281,"in_reply_to":"5a710552_7f17aea3","updated":"2015-12-16 08:39:46.000000000","message":"Got it. I still need to look at that more deeply but this code looks good and test also. Thanks for explanation :)","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"05072c6ebc593f192180cf753431da44ca10aa21","unresolved":false,"context_lines":[{"line_number":3278,"context_line":""},{"line_number":3279,"context_line":"    def test_durable_timestamp_not_open(self):"},{"line_number":3280,"context_line":"        df \u003d self._simple_get_diskfile()"},{"line_number":3281,"context_line":"        self.assertIsNone(df.durable_timestamp)"},{"line_number":3282,"context_line":""},{"line_number":3283,"context_line":"    def test_error_in_hash_cleanup_listdir(self):"},{"line_number":3284,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_7f17aea3","line":3281,"in_reply_to":"7a740942_e2e283d4","updated":"2015-12-15 15:36:50.000000000","message":"Done.\n\nThis behaviour is deliberately a little different to DiskFile.timestamp and DiskFile.data_timestamp i.e. the durable timestamp *can* be queried even when diskfile open() failed - I use that behaviour in a dependent patch:\n\nhttps://review.openstack.org/#/c/232684/6/swift/obj/ssync_receiver.py line 270\n\nwhere the diskfile may fail to open, but we need to know what (if any) durable timestamp it has.\n\nBut I will change so that open() must have at least been called (even if it fails) otherwise DikFileNotOpen is raised.","commit_id":"7fd2963ffe9e2caf09ec4100cd2d9b99148a4d60"}]}