)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":13,"context_line":"the receiver"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"- guts of reconstructor sorta follow replicator but there\u0027s not a"},{"line_number":16,"context_line":"notiong of update() and upate_deleted().  There is a single job"},{"line_number":17,"context_line":"processor and jobs are processed partition by partition.  At the"},{"line_number":18,"context_line":"end of processing a partition, the ECrecon will try to remove it"},{"line_number":19,"context_line":"and it may or may not work depending on whether everything was"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":44,"id":"da9b358b_5b31c8a1","line":16,"updated":"2015-03-30 06:14:49.000000000","message":"s/notiong/notion/g \n\ncommit messages are important, but seeing as this needs to merge soon I\u0027m ok with this being a NIT if no other patchsets are raised, otherwise fix it.","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"}],"etc/object-server.conf-sample":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":146,"context_line":"#recon_cache_path \u003d /var/cache/swift"},{"line_number":147,"context_line":"#recon_lock_path \u003d /var/lock"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"# TODO:  document this section"},{"line_number":150,"context_line":"[object-reconstructor]"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"[object-replicator]"}],"source_content_type":"application/octet-stream","patch_set":44,"id":"da9b358b_db1dd810","line":149,"updated":"2015-03-30 06:14:49.000000000","message":"Do we want to have todo\u0027s in sample config files that users will see. Maybe we need to have comments on how EC and therefore the object-reconstructor is in  beta (USE IN PRODUCTION AT YOUR OWN RISK).","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"}],"swift/obj/diskfile.py":[{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"070cdd465e78a684b5d764d4f84eeaa1b6300fae","unresolved":false,"context_lines":[{"line_number":90,"context_line":"# now seeing all I really need is a one old style hashes.pkl"},{"line_number":91,"context_line":"# dict and one suff: FI dict.  Will hold off doing this until"},{"line_number":92,"context_line":"# a bit later though (after reconstruction is done)"},{"line_number":93,"context_line":"class SuffixHashes(object):"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"    def __init__(self, hashes_file\u003dNone, hashes_dict\u003dNone):"},{"line_number":96,"context_line":"        self.by_suffix \u003d {}"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3c543653","line":93,"updated":"2015-01-06 00:38:54.000000000","message":"please put a docstring with a sample of what this dict looks like","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"070cdd465e78a684b5d764d4f84eeaa1b6300fae","unresolved":false,"context_lines":[{"line_number":91,"context_line":"# dict and one suff: FI dict.  Will hold off doing this until"},{"line_number":92,"context_line":"# a bit later though (after reconstruction is done)"},{"line_number":93,"context_line":"class SuffixHashes(object):"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"    def __init__(self, hashes_file\u003dNone, hashes_dict\u003dNone):"},{"line_number":96,"context_line":"        self.by_suffix \u003d {}"},{"line_number":97,"context_line":"        self.by_frag_index \u003d {}"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_f7d80755","line":94,"updated":"2015-01-06 00:38:54.000000000","message":"see https://gist.github.com/notmyname/68874a5c7b5c0cb6d79e for an alternate way to do self.by_frag_index","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"ff58e1a1029eb14bde14ce814c0a870e3c88bd74","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                for suff in self.by_frag_index.get(frag_index).keys():"},{"line_number":195,"context_line":"                    if suff \u003d\u003d suffix:"},{"line_number":196,"context_line":"                        self.by_frag_index.get(frag_index)[suff] \u003d None"},{"line_number":197,"context_line":"        return changed"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"def _get_filename(fd):"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3ccef683","line":197,"updated":"2015-01-06 00:04:52.000000000","message":"def invalidate(self, suffix):\n        changed \u003d False\n        frag_indexes \u003d self.by_suffix.get(suffix)\n        if frag_indexes is not None:\n            for frag_index in frag_indexes:\n                self.by_suffix[suffix][frag_index] \u003d None\n                if frag_index in self.by_frag_index:\n                    self.by_frag_index[frag_index][suffix] \u003d None\n                changed \u003d True\n        return changed","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                for suff in self.by_frag_index.get(frag_index).keys():"},{"line_number":195,"context_line":"                    if suff \u003d\u003d suffix:"},{"line_number":196,"context_line":"                        self.by_frag_index.get(frag_index)[suff] \u003d None"},{"line_number":197,"context_line":"        return changed"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"def _get_filename(fd):"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_779337a3","line":197,"in_reply_to":"3a961159_1c005a65","updated":"2015-01-06 01:42:10.000000000","message":"if not frag_indexes:\n    return false\nfor frag_index, hash_ in frag_indexes.items():\n    if hash_ is None:\n        continue\n    frag_indexes[frag_index] \u003d None\n    changed \u003d True","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"401212581494d3c2adf0fdafe1ef2b99cfa2cfd2","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                for suff in self.by_frag_index.get(frag_index).keys():"},{"line_number":195,"context_line":"                    if suff \u003d\u003d suffix:"},{"line_number":196,"context_line":"                        self.by_frag_index.get(frag_index)[suff] \u003d None"},{"line_number":197,"context_line":"        return changed"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"def _get_filename(fd):"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_1c005a65","line":197,"in_reply_to":"3a961159_1cb21ae5","updated":"2015-01-06 00:12:01.000000000","message":"def invalidate(self, suffix):\n        changed \u003d False\n        frag_indexes \u003d self.by_suffix.get(suffix)\n        if frag_indexes is not None:\n            for frag_index in frag_indexes:\n                if frag_indexes[frag_index] is not None:\n                    frag_indexes[frag_index] \u003d None\n                    changed \u003d True\n                    if frag_index in self.by_frag_index:\n                        self.by_frag_index[frag_index][suffix] \u003d None\n        return changed","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"aa72ea7fae3ad227737bc40ed6b7f0221b49bebd","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                for suff in self.by_frag_index.get(frag_index).keys():"},{"line_number":195,"context_line":"                    if suff \u003d\u003d suffix:"},{"line_number":196,"context_line":"                        self.by_frag_index.get(frag_index)[suff] \u003d None"},{"line_number":197,"context_line":"        return changed"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"def _get_filename(fd):"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_1cb21ae5","line":197,"in_reply_to":"3a961159_3ccef683","updated":"2015-01-06 00:09:39.000000000","message":"def invalidate(self, suffix):\n        changed \u003d False\n        frag_indexes \u003d self.by_suffix.get(suffix)\n        if frag_indexes is not None:\n            for frag_index in frag_indexes:\n                if self.by_suffix[suffix][frag_index] is not None:\n                    self.by_suffix[suffix][frag_index] \u003d None\n                    changed \u003d True\n                    if frag_index in self.by_frag_index:\n                        self.by_frag_index[frag_index][suffix] \u003d None\n        return changed","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":413,"context_line":"    :raises PathNotDir: if given path is not a valid directory"},{"line_number":414,"context_line":"    :raises OSError: for non-ENOTDIR errors"},{"line_number":415,"context_line":"    \"\"\""},{"line_number":416,"context_line":"    hashers \u003d hash_per_fi \u003d {}"},{"line_number":417,"context_line":"    try:"},{"line_number":418,"context_line":"        path_contents \u003d sorted(os.listdir(path))"},{"line_number":419,"context_line":"    except OSError as err:"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_595bd453","line":416,"updated":"2015-01-05 23:57:25.000000000","message":"these are the same object, not two new dicts","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":413,"context_line":"    :raises PathNotDir: if given path is not a valid directory"},{"line_number":414,"context_line":"    :raises OSError: for non-ENOTDIR errors"},{"line_number":415,"context_line":"    \"\"\""},{"line_number":416,"context_line":"    hashers \u003d hash_per_fi \u003d {}"},{"line_number":417,"context_line":"    try:"},{"line_number":418,"context_line":"        path_contents \u003d sorted(os.listdir(path))"},{"line_number":419,"context_line":"    except OSError as err:"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3bb58e32","line":416,"in_reply_to":"3a961159_595bd453","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":453,"context_line":"    except OSError:"},{"line_number":454,"context_line":"        pass"},{"line_number":455,"context_line":"    for frag_index in hashers:"},{"line_number":456,"context_line":"        hash_per_fi[frag_index] \u003d hashers[frag_index].hexdigest()"},{"line_number":457,"context_line":"    return hash_per_fi"},{"line_number":458,"context_line":""},{"line_number":459,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_392528c2","line":456,"updated":"2015-01-05 23:57:25.000000000","message":"of course, you over-write *all* the keys here, so it works out in the end - you could even just have the one dict I think?","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":453,"context_line":"    except OSError:"},{"line_number":454,"context_line":"        pass"},{"line_number":455,"context_line":"    for frag_index in hashers:"},{"line_number":456,"context_line":"        hash_per_fi[frag_index] \u003d hashers[frag_index].hexdigest()"},{"line_number":457,"context_line":"    return hash_per_fi"},{"line_number":458,"context_line":""},{"line_number":459,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_5bb25a4a","line":456,"in_reply_to":"3a961159_392528c2","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":501,"context_line":"    suffix_hashes \u003d SuffixHashes(hashes_file)"},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"    # if a new suffix dir is found add it without any FI or hash info"},{"line_number":504,"context_line":"    if do_listdir or suffix_hashes.file_error:"},{"line_number":505,"context_line":"        for suffix in os.listdir(partition_dir):"},{"line_number":506,"context_line":"            if len(suffix) \u003d\u003d 3:"},{"line_number":507,"context_line":"                suffix_hashes.add(suffix)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_06ce4b2d","line":504,"updated":"2015-01-05 23:57:25.000000000","message":"i\u0027m pretty meh on .file_error","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":501,"context_line":"    suffix_hashes \u003d SuffixHashes(hashes_file)"},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"    # if a new suffix dir is found add it without any FI or hash info"},{"line_number":504,"context_line":"    if do_listdir or suffix_hashes.file_error:"},{"line_number":505,"context_line":"        for suffix in os.listdir(partition_dir):"},{"line_number":506,"context_line":"            if len(suffix) \u003d\u003d 3:"},{"line_number":507,"context_line":"                suffix_hashes.add(suffix)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_1b8ff277","line":504,"in_reply_to":"3a961159_06ce4b2d","updated":"2015-01-19 23:59:55.000000000","message":"OK, not sure if you just mean the name or what but we can cover it in a later review I think.","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":513,"context_line":""},{"line_number":514,"context_line":"    # loop through all suffixes needing hash recalc"},{"line_number":515,"context_line":"    for suffix, hash_dict in suffix_hashes.by_suffix.items():"},{"line_number":516,"context_line":"        if not [h for h in hash_dict if hash_dict[h] is not None]:"},{"line_number":517,"context_line":"            suffix_dir \u003d join(partition_dir, suffix)"},{"line_number":518,"context_line":"            try:"},{"line_number":519,"context_line":"                # hash suffix will give us a hash per fragment"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_d9650446","line":516,"updated":"2015-01-05 23:57:25.000000000","message":"if any(h is None for h in hash_dict.values()):\n        ... do recalc","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":513,"context_line":""},{"line_number":514,"context_line":"    # loop through all suffixes needing hash recalc"},{"line_number":515,"context_line":"    for suffix, hash_dict in suffix_hashes.by_suffix.items():"},{"line_number":516,"context_line":"        if not [h for h in hash_dict if hash_dict[h] is not None]:"},{"line_number":517,"context_line":"            suffix_dir \u003d join(partition_dir, suffix)"},{"line_number":518,"context_line":"            try:"},{"line_number":519,"context_line":"                # hash suffix will give us a hash per fragment"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_9bea020d","line":516,"in_reply_to":"3a961159_d9650446","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":522,"context_line":"                                          policy_idx)"},{"line_number":523,"context_line":"                # add/update suffix_hashes with all the info"},{"line_number":524,"context_line":"                for fi in hash_per_fi.keys():"},{"line_number":525,"context_line":"                    suffix_hashes.add(suffix, fi, hash_per_fi[fi])"},{"line_number":526,"context_line":"                hashed +\u003d 1"},{"line_number":527,"context_line":"            except PathNotDir:"},{"line_number":528,"context_line":"                suffix_hashes.delete(suffix)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_bcff4600","line":525,"updated":"2015-01-05 23:57:25.000000000","message":"for fi, hashes in hash_per_fi.items():\n        suffix_hashes.add(suffix, fi, hashes)","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":522,"context_line":"                                          policy_idx)"},{"line_number":523,"context_line":"                # add/update suffix_hashes with all the info"},{"line_number":524,"context_line":"                for fi in hash_per_fi.keys():"},{"line_number":525,"context_line":"                    suffix_hashes.add(suffix, fi, hash_per_fi[fi])"},{"line_number":526,"context_line":"                hashed +\u003d 1"},{"line_number":527,"context_line":"            except PathNotDir:"},{"line_number":528,"context_line":"                suffix_hashes.delete(suffix)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3be3ce35","line":525,"in_reply_to":"3a961159_bcff4600","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a98d31ceb6b20782c198b189a47bf7e19667d040","unresolved":false,"context_lines":[{"line_number":532,"context_line":""},{"line_number":533,"context_line":"    if modified:"},{"line_number":534,"context_line":"        with lock_path(partition_dir):"},{"line_number":535,"context_line":"            if suffix_hashes.file_error or not exists(hashes_file) or \\"},{"line_number":536,"context_line":"                    getmtime(hashes_file) \u003d\u003d suffix_hashes.mtime:"},{"line_number":537,"context_line":"                suffix_hashes.serialize(partition_dir)"},{"line_number":538,"context_line":"                return hashed, suffix_hashes"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_7c307ee9","line":535,"updated":"2015-01-05 23:57:25.000000000","message":"oh hrm... so here is .file_error again, instead of the force_rewrite state... so maybe it\u0027s ok.","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":532,"context_line":""},{"line_number":533,"context_line":"    if modified:"},{"line_number":534,"context_line":"        with lock_path(partition_dir):"},{"line_number":535,"context_line":"            if suffix_hashes.file_error or not exists(hashes_file) or \\"},{"line_number":536,"context_line":"                    getmtime(hashes_file) \u003d\u003d suffix_hashes.mtime:"},{"line_number":537,"context_line":"                suffix_hashes.serialize(partition_dir)"},{"line_number":538,"context_line":"                return hashed, suffix_hashes"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_5be09a26","line":535,"in_reply_to":"3a961159_7c307ee9","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"86c6fa643ef58de375ed3b4d39d8c858d7d58a95","unresolved":false,"context_lines":[{"line_number":115,"context_line":"    def mtime(self, value):"},{"line_number":116,"context_line":"        self._mtime \u003d value"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"    def serialize(self, partition_dir):"},{"line_number":119,"context_line":"        # we don\u0027t store the redundant frag_index dict"},{"line_number":120,"context_line":"        self.by_frag_index \u003d {}"},{"line_number":121,"context_line":"        write_pickle("}],"source_content_type":"text/x-python","patch_set":14,"id":"3a961159_4f2ccd4a","line":118,"updated":"2015-01-06 17:59:11.000000000","message":"make this conditional on pol idx and write old format for repl type and new format for EC type to support downgraded w/o breaking repl","commit_id":"38ce6e1ce2593f6efd6b02986e66ee64c82538ac"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"9ff16be42011355efe11ad6477d5c0fbb9772c1c","unresolved":false,"context_lines":[{"line_number":115,"context_line":"    def mtime(self, value):"},{"line_number":116,"context_line":"        self._mtime \u003d value"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"    def serialize(self, partition_dir):"},{"line_number":119,"context_line":"        # we don\u0027t store the redundant frag_index dict"},{"line_number":120,"context_line":"        self.by_frag_index \u003d {}"},{"line_number":121,"context_line":"        write_pickle("}],"source_content_type":"text/x-python","patch_set":14,"id":"3a961159_fbc9467b","line":118,"in_reply_to":"3a961159_4f2ccd4a","updated":"2015-01-20 00:02:51.000000000","message":"Done","commit_id":"38ce6e1ce2593f6efd6b02986e66ee64c82538ac"}],"swift/obj/reconstructor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":181,"context_line":"        # the fragment index we need to reconstruct is the position index"},{"line_number":182,"context_line":"        # of the node we\u0027re rebuiling to within the primart part list"},{"line_number":183,"context_line":"        fi_to_rebuild \u003d [index for index, n in enumerate(part_nodes)"},{"line_number":184,"context_line":"                         if n[\u0027id\u0027] \u003d\u003d node[\u0027id\u0027]]"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"        print \"(%s) part_nodes IS %s \" % (os.getpid(), part_nodes)"},{"line_number":187,"context_line":"        print \"(%s) node to rebuild TO IS %s \" % (os.getpid(), node)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_d21239b3","line":184,"updated":"2015-01-06 01:42:10.000000000","message":"node[\u0027index\u0027] would do","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":181,"context_line":"        # the fragment index we need to reconstruct is the position index"},{"line_number":182,"context_line":"        # of the node we\u0027re rebuiling to within the primart part list"},{"line_number":183,"context_line":"        fi_to_rebuild \u003d [index for index, n in enumerate(part_nodes)"},{"line_number":184,"context_line":"                         if n[\u0027id\u0027] \u003d\u003d node[\u0027id\u0027]]"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"        print \"(%s) part_nodes IS %s \" % (os.getpid(), part_nodes)"},{"line_number":187,"context_line":"        print \"(%s) node to rebuild TO IS %s \" % (os.getpid(), node)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_fbbf86fb","line":184,"in_reply_to":"3a961159_d21239b3","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":253,"context_line":"                writer.put(metadata)"},{"line_number":254,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":255,"context_line":"            self.logger.exception(_(\"Error (reconstructor) attempting to \""},{"line_number":256,"context_line":"                                    \"write local temporary diskfile\"))"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"        # return the df and its temp location to ssync who will deliver the"},{"line_number":259,"context_line":"        # file and cleanup the temp"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_f21d7574","line":256,"updated":"2015-01-06 01:42:10.000000000","message":"ok yeah, so this is where we storage a copy of the rebuilt fragment index in the nodes local /tmp file system before handing over to ssync to ship over the write.  ssync_sender.send_put shows how much of the diskfile interface we\u0027d need to fake out to in order to hand it a thing that quacks like a diskfile but actually get\u0027s it\u0027s iterable bits from sockets.","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":253,"context_line":"                writer.put(metadata)"},{"line_number":254,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":255,"context_line":"            self.logger.exception(_(\"Error (reconstructor) attempting to \""},{"line_number":256,"context_line":"                                    \"write local temporary diskfile\"))"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"        # return the df and its temp location to ssync who will deliver the"},{"line_number":259,"context_line":"        # file and cleanup the temp"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_bb3dfe90","line":256,"in_reply_to":"3a961159_f21d7574","updated":"2015-01-19 23:59:55.000000000","message":"Done","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":380,"context_line":"            while attempts_left \u003e 0:"},{"line_number":381,"context_line":"                # If this throws StopIteration it will be caught way below"},{"line_number":382,"context_line":"                node \u003d next(nodes)"},{"line_number":383,"context_line":"                node_position \u003d str(part_nodes.index(node))"},{"line_number":384,"context_line":"                print \"(%s) NEW PASS: LOCAL INDEX DICT %s \" % (os.getpid(),"},{"line_number":385,"context_line":"                                                               job[\u0027indexes\u0027])"},{"line_number":386,"context_line":"                print \"(%s) JOB %s \" % (os.getpid(), job)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_57b0fb8a","line":383,"updated":"2015-01-06 01:42:10.000000000","message":"use node[\u0027index\u0027] and get rid of the state var","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":380,"context_line":"            while attempts_left \u003e 0:"},{"line_number":381,"context_line":"                # If this throws StopIteration it will be caught way below"},{"line_number":382,"context_line":"                node \u003d next(nodes)"},{"line_number":383,"context_line":"                node_position \u003d str(part_nodes.index(node))"},{"line_number":384,"context_line":"                print \"(%s) NEW PASS: LOCAL INDEX DICT %s \" % (os.getpid(),"},{"line_number":385,"context_line":"                                                               job[\u0027indexes\u0027])"},{"line_number":386,"context_line":"                print \"(%s) JOB %s \" % (os.getpid(), job)"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3b510eda","line":383,"in_reply_to":"3a961159_57b0fb8a","updated":"2015-01-19 23:59:55.000000000","message":"this func is gone, but will make sure to use node[\u0027index\u0027] moving forward","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":407,"context_line":"                        del resp"},{"line_number":408,"context_line":"                    suffixes \u003d get_suffix_delta(local_hash.by_suffix,"},{"line_number":409,"context_line":"                                                remote_hash.by_suffix,"},{"line_number":410,"context_line":"                                                job[\u0027indexes\u0027],"},{"line_number":411,"context_line":"                                                node_position)"},{"line_number":412,"context_line":"                    if not suffixes:"},{"line_number":413,"context_line":"                        print \"(%s) UPDATE() says: noting to see here...\" \\"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_b7ff5f01","line":410,"updated":"2015-01-06 01:42:10.000000000","message":"instead of passing in indexes so that get_suffix_delta can peek into indexes[\u0027local\u0027], just pass in this jobs local frag_index.\n\nso:\n\nsuffix_diffs \u003d get_suffix_delta(local_frag_index, local_suffixes, remote_frag_index (i.e. node_position), remote_suffixes)","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":407,"context_line":"                        del resp"},{"line_number":408,"context_line":"                    suffixes \u003d get_suffix_delta(local_hash.by_suffix,"},{"line_number":409,"context_line":"                                                remote_hash.by_suffix,"},{"line_number":410,"context_line":"                                                job[\u0027indexes\u0027],"},{"line_number":411,"context_line":"                                                node_position)"},{"line_number":412,"context_line":"                    if not suffixes:"},{"line_number":413,"context_line":"                        print \"(%s) UPDATE() says: noting to see here...\" \\"}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_db59eafe","line":410,"in_reply_to":"3a961159_b7ff5f01","updated":"2015-01-19 23:59:55.000000000","message":"this is all torn apart and WIP so will try to incorporate this when its tackled.  Several other things are up first","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":547,"context_line":"            # didn\u0027t find local_id in part_nodes"},{"line_number":548,"context_line":"            handoff_node \u003d True"},{"line_number":549,"context_line":"            partners \u003d []"},{"line_number":550,"context_line":"        return handoff_node, partners, indexes"},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"    def build_reconstruction_jobs(self, policy, jobs, ips):"},{"line_number":553,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_92727148","line":550,"updated":"2015-01-06 01:42:10.000000000","message":"this is getting hairy, at worst we could to return a dict instead of big long tuple, at best we can reduce what actually needs to be returned.","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":547,"context_line":"            # didn\u0027t find local_id in part_nodes"},{"line_number":548,"context_line":"            handoff_node \u003d True"},{"line_number":549,"context_line":"            partners \u003d []"},{"line_number":550,"context_line":"        return handoff_node, partners, indexes"},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"    def build_reconstruction_jobs(self, policy, jobs, ips):"},{"line_number":553,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_7b01d6b4","line":550,"in_reply_to":"3a961159_92727148","updated":"2015-01-19 23:59:55.000000000","message":"indexes is no longer used in the latest ver but will follow the if \u003e 2 use a dict guideline moving forward elsewhere","commit_id":"78c9d318a2469e9c3eb2f6e3d21bf6170b72a29e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5d34c07267a53aea5c07f94f9cf6b6717e9d5420","unresolved":false,"context_lines":[{"line_number":212,"context_line":"                self.metadata \u003d metadata"},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"                # the new FA is going to have the same len as others in the set"},{"line_number":215,"context_line":"                self._content_length \u003d self.metadata[\u0027Content-Length\u0027]"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"                # update the FI and delete the ETag, the obj server will"},{"line_number":218,"context_line":"                # recalc on the other side..."}],"source_content_type":"text/x-python","patch_set":15,"id":"3a961159_fffde43b","line":215,"updated":"2015-01-08 17:01:23.000000000","message":"word on the street is all the fa\u0027s are the same size except for the last?","commit_id":"7a1d70a9fa4ef2359c8678fcaa5a12bec0c8a9f4"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"e7b5b49aac48ebc418f737a7ecdef098f0f6e441","unresolved":false,"context_lines":[{"line_number":212,"context_line":"                self.metadata \u003d metadata"},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"                # the new FA is going to have the same len as others in the set"},{"line_number":215,"context_line":"                self._content_length \u003d self.metadata[\u0027Content-Length\u0027]"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"                # update the FI and delete the ETag, the obj server will"},{"line_number":218,"context_line":"                # recalc on the other side..."}],"source_content_type":"text/x-python","patch_set":15,"id":"3a961159_d6571126","line":215,"in_reply_to":"3a961159_fffde43b","updated":"2015-01-09 19:17:44.000000000","message":"they\u0027re always\u0027s the same size.  Its the last segment in an archive that may not be the same size as all the other segments but that segment is the same size for all FA\u0027s so its always the case that every FA is the same size.\n\nnote that ssync already uses chunked xfers however the subreq_uter() used in processing the subrequest at the remote object server is written assuming the length is known (because it always would be for regular ssync work) so instead of rewriting that iter to handle unknown length I simply provide the length just as with any other object that ssync is sending over... pretty cool.","commit_id":"7a1d70a9fa4ef2359c8678fcaa5a12bec0c8a9f4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be6df71f87fccf467e01c96e4d4ab5754214880f","unresolved":false,"context_lines":[{"line_number":200,"context_line":"        from_nodes \u003d []"},{"line_number":201,"context_line":"        for frag_num in fragments_needed:"},{"line_number":202,"context_line":"            from_nodes.append([n for n in part_nodes"},{"line_number":203,"context_line":"                              if n[\u0027index\u0027] \u003d\u003d frag_num][0])"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"        # get the archives we need"},{"line_number":206,"context_line":"        headers \u003d {}"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_e6afbc29","line":203,"updated":"2015-02-09 20:06:56.000000000","message":"[n for n in part_nodes if n[\u0027index\u0027] in fragments_needed]","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be6df71f87fccf467e01c96e4d4ab5754214880f","unresolved":false,"context_lines":[{"line_number":467,"context_line":"                    tpool.execute(self.tpool_get_info, job[\u0027path\u0027],"},{"line_number":468,"context_line":"                                  job[\u0027policy_idx\u0027],"},{"line_number":469,"context_line":"                                  recalculate\u003dsuffixes,"},{"line_number":470,"context_line":"                                  reclaim_age\u003dself.reclaim_age)"},{"line_number":471,"context_line":"                self.suffix_count +\u003d len(local_suff)"},{"line_number":472,"context_line":"                try:"},{"line_number":473,"context_line":"                    os.rmdir(job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_0b84d9af","line":470,"updated":"2015-02-09 20:06:56.000000000","message":"I don\u0027t want to have to recalculate hashes a *third* time, I know that deleting some stuff may invalidate a hash and i\u0027m sure we\u0027ll have to figure out something, but it\u0027d be nice if we could do it after all the jobs for the part (each fi) finish.","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"6025d520e5e61199ff785426ab94f2a7832a38b7","unresolved":false,"context_lines":[{"line_number":467,"context_line":"                    tpool.execute(self.tpool_get_info, job[\u0027path\u0027],"},{"line_number":468,"context_line":"                                  job[\u0027policy_idx\u0027],"},{"line_number":469,"context_line":"                                  recalculate\u003dsuffixes,"},{"line_number":470,"context_line":"                                  reclaim_age\u003dself.reclaim_age)"},{"line_number":471,"context_line":"                self.suffix_count +\u003d len(local_suff)"},{"line_number":472,"context_line":"                try:"},{"line_number":473,"context_line":"                    os.rmdir(job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_bcef5d1d","line":470,"in_reply_to":"da86d52c_0b84d9af","updated":"2015-02-10 20:44:37.000000000","message":"agreed, will mark a TODO to followup on recalcing remote hash post delete","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be6df71f87fccf467e01c96e4d4ab5754214880f","unresolved":false,"context_lines":[{"line_number":646,"context_line":"        ips \u003d whataremyips()"},{"line_number":647,"context_line":"        for policy in POLICIES:"},{"line_number":648,"context_line":"            if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":649,"context_line":"                jobs \u003d self.build_reconstruction_jobs(policy, ips)"},{"line_number":650,"context_line":"        random.shuffle(jobs)"},{"line_number":651,"context_line":"        if self.handoffs_first:"},{"line_number":652,"context_line":"            # Move the handoff revert jobs to the front of the list"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_66d02c54","line":649,"updated":"2015-02-09 20:06:56.000000000","message":"the job list is the top loop in reconstruct, in replication land it\u0027s per partition.  I\u0027d like to keep the top level loop in reconstruct at a cardinality of \u003d\u003d partitions and save the build_reconstruction_jobs explosion to just be per part.\n\n    for p in parts:\n        jobs \u003d build_reconstruction_jobs()\n        for j in jobs:\n             process_job(j)\n        clean_up_suffixes()","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":163,"context_line":"        part_nodes \u003d job[\u0027object_ring\u0027].get_part_nodes(int(job[\u0027partition\u0027]))"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        # the fragment index we need to reconstruct is the position index"},{"line_number":166,"context_line":"        # of the node we\u0027re rebuiling to within the primart part list"},{"line_number":167,"context_line":"        fi_to_rebuild \u003d [node[\u0027index\u0027]]"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"        # ask the library which fragemnts we need"}],"source_content_type":"text/x-python","patch_set":44,"id":"da9b358b_7b4a4c85","line":166,"updated":"2015-03-30 06:14:49.000000000","message":"s/primart/primary/","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":202,"context_line":"        if len(responses) \u003c len(fragments_needed):"},{"line_number":203,"context_line":"            return None"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"        class DiskFileLikeThing(object):"},{"line_number":206,"context_line":"            def __init__(self, metadata):"},{"line_number":207,"context_line":"                # start with metadata from a participating FA"},{"line_number":208,"context_line":"                self.metadata \u003d metadata"}],"source_content_type":"text/x-python","patch_set":44,"id":"da9b358b_ce3754e2","line":205,"updated":"2015-03-30 06:14:49.000000000","message":"NIT: Maybe a better name then DiskfileLikeThing, although I think it\u0027s funny :)\n\nReconstructDiskFile or something?","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        #   here we sync with our partners"},{"line_number":503,"context_line":"        # 2) part dir with one local and mix of others"},{"line_number":504,"context_line":"        #   here we need to sync with our partners where FI matches"},{"line_number":505,"context_line":"        #   the lcoal_id  , all others are sync\u0027d with their home"},{"line_number":506,"context_line":"        #   nodes and then killed"},{"line_number":507,"context_line":"        # 3) part dir with no local FI and just one or more others"},{"line_number":508,"context_line":"        #   here we sync with just the FI that exists, nobody else"}],"source_content_type":"text/x-python","patch_set":44,"id":"da9b358b_ae1ad015","line":505,"updated":"2015-03-30 06:14:49.000000000","message":"s/lcoal_id/local_id/","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":518,"context_line":"                job_info \u003d {}"},{"line_number":519,"context_line":"                job_info[\u0027suffix\u0027] \u003d suffix"},{"line_number":520,"context_line":"                job_info[\u0027frag_index\u0027] \u003d frag_index"},{"line_number":521,"context_line":"                job_info[\u0027sync_to\u0027] \u003d []"},{"line_number":522,"context_line":""},{"line_number":523,"context_line":"                # tombstones/durables have no FI encoded so when we"},{"line_number":524,"context_line":"                # build a job for them, we can\u0027t know where to send"}],"source_content_type":"text/x-python","patch_set":44,"id":"da9b358b_4ea6a4c2","line":521,"updated":"2015-03-30 06:14:49.000000000","message":"NIT: Not that this matters, but rather then creating a dict, and then populating it one entry at a time, this would be a good case to just:\n\n  job_info \u003d {\n      \u0027suffix\u0027: suffix,\n      \u0027frag_index\u0027: frag_index,\n      \u0027sync_to\u0027: []\n  }","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"816cc380b68e8f578b9590a1f25a3652deb33cc7","unresolved":false,"context_lines":[{"line_number":522,"context_line":""},{"line_number":523,"context_line":"                # tombstones/durables have no FI encoded so when we"},{"line_number":524,"context_line":"                # build a job for them, we can\u0027t know where to send"},{"line_number":525,"context_line":"                # them unless we\u0027re not a handoff.  So, if we\u0027re"},{"line_number":526,"context_line":"                # reverting we talk to all part_nodes, otherwise"},{"line_number":527,"context_line":"                # we talk to just the partners"},{"line_number":528,"context_line":"                if frag_index is None:"}],"source_content_type":"text/x-python","patch_set":44,"id":"da9b358b_2e99206f","line":525,"updated":"2015-03-30 06:14:49.000000000","message":"Lol, just a few negatives in that sentence \u0027we can\u0027t know where to send them unless we\u0027re not a handoff\u0027. So this means we only know where to send them if they are primaries?","commit_id":"936e4e8ab7ac6e0992b05276c82ba72701cf1209"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":166,"context_line":"    def _get_response(self, node, part, path, headers, policy):"},{"line_number":167,"context_line":"        \"\"\""},{"line_number":168,"context_line":"        Helper function for reconstruction that GETs a single EC fragment"},{"line_number":169,"context_line":"        archive"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"        :param node: the node to GET from"},{"line_number":172,"context_line":"        :param part: the partition"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_180af421","line":169,"updated":"2015-03-31 09:58:16.000000000","message":"(nits) helper \"method\"","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":166,"context_line":"    def _get_response(self, node, part, path, headers, policy):"},{"line_number":167,"context_line":"        \"\"\""},{"line_number":168,"context_line":"        Helper function for reconstruction that GETs a single EC fragment"},{"line_number":169,"context_line":"        archive"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"        :param node: the node to GET from"},{"line_number":172,"context_line":"        :param part: the partition"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_e47af0f0","line":169,"in_reply_to":"da9b358b_180af421","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":196,"context_line":""},{"line_number":197,"context_line":"    def reconstruct_fa(self, job, node, policy, metadata):"},{"line_number":198,"context_line":"        \"\"\""},{"line_number":199,"context_line":"        Reconstructs a fragment archive"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        :param job: job from ssync_sender"},{"line_number":202,"context_line":"        :param node: node that we\u0027re rebuilding to"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_f85bd0f4","line":199,"updated":"2015-03-31 09:58:16.000000000","message":"To be more helpful for developers, I would love to write down similar message from commit log like as \"this function will be called from ssync\"","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":196,"context_line":""},{"line_number":197,"context_line":"    def reconstruct_fa(self, job, node, policy, metadata):"},{"line_number":198,"context_line":"        \"\"\""},{"line_number":199,"context_line":"        Reconstructs a fragment archive"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"        :param job: job from ssync_sender"},{"line_number":202,"context_line":"        :param node: node that we\u0027re rebuilding to"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_a4a06844","line":199,"in_reply_to":"da9b358b_f85bd0f4","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":205,"context_line":"        :returns: a DiskFile like class for use by ssync"},{"line_number":206,"context_line":"        \"\"\""},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"        part_nodes \u003d job[\u0027object_ring\u0027].get_part_nodes(int(job[\u0027partition\u0027]))"},{"line_number":209,"context_line":"        part_nodes.remove(node)"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        # the fragment index we need to reconstruct is the position index"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_388c9834","line":208,"updated":"2015-03-31 09:58:16.000000000","message":"2 comments:\n\n- job[\u0027object_ring\u0027] is job[\u0027policy\u0027].object_ring? I didn\u0027t find the assignment the ring to the dict.\n\n- No need to cast the job[\u0027partition\u0027] to int because it has been already casted on job building. (See L664 on this file)","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":205,"context_line":"        :returns: a DiskFile like class for use by ssync"},{"line_number":206,"context_line":"        \"\"\""},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"        part_nodes \u003d job[\u0027object_ring\u0027].get_part_nodes(int(job[\u0027partition\u0027]))"},{"line_number":209,"context_line":"        part_nodes.remove(node)"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        # the fragment index we need to reconstruct is the position index"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_245a1850","line":208,"in_reply_to":"da9b358b_388c9834","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":209,"context_line":"        part_nodes.remove(node)"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        # the fragment index we need to reconstruct is the position index"},{"line_number":212,"context_line":"        # of the node we\u0027re rebuiling to within the primart part list"},{"line_number":213,"context_line":"        fi_to_rebuild \u003d node[\u0027index\u0027]"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"        # KISS send out connection requests to all nodes, see what sticks"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_18d2f4d2","line":212,"updated":"2015-03-31 09:58:16.000000000","message":"(nits) the primary","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":209,"context_line":"        part_nodes.remove(node)"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        # the fragment index we need to reconstruct is the position index"},{"line_number":212,"context_line":"        # of the node we\u0027re rebuiling to within the primart part list"},{"line_number":213,"context_line":"        fi_to_rebuild \u003d node[\u0027index\u0027]"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"        # KISS send out connection requests to all nodes, see what sticks"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_642700cb","line":212,"in_reply_to":"da9b358b_18d2f4d2","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":253,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":254,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":255,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"},{"line_number":256,"context_line":"        return policy.pyeclib_driver.encode(segment)[fragment_index]"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"    def make_rebuilt_fragment_iter(self, responses, policy, fragment_index):"},{"line_number":259,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_7802200d","line":256,"updated":"2015-03-31 09:58:16.000000000","message":"Do you have any plan to use pyeclib reconstruct on this patch set? Or future work?","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":253,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":254,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":255,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"},{"line_number":256,"context_line":"        return policy.pyeclib_driver.encode(segment)[fragment_index]"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"    def make_rebuilt_fragment_iter(self, responses, policy, fragment_index):"},{"line_number":259,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_ef2769b4","line":256,"in_reply_to":"da9b358b_7802200d","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":404,"context_line":"                        sub_dict_remote.get(remote_index, \u0027-1\u0027):"},{"line_number":405,"context_line":"                    suffixes.append(suffix)"},{"line_number":406,"context_line":"            except KeyError:"},{"line_number":407,"context_line":"                suffixes.append(suffix)"},{"line_number":408,"context_line":"        return suffixes"},{"line_number":409,"context_line":""},{"line_number":410,"context_line":"    def process_job(self, job):"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_56516d0d","line":407,"updated":"2015-03-31 09:58:16.000000000","message":"Not sure, but we can make it to be slim code like as follows:\n\nfor suffix, sub_dict_local in local_suff.iteritems():\n   sub_dict_remote \u003d remote_suff.get(suffix, {})\n   if local_index not in sub_dict_local or \\\n       sub_dict_local[local_index] !\u003d sub_dict_remote.get(remote_index)\n     suffixes.append(suffix)\n   else:\n     # no longer need to this else, exist just to mean nothing to do\n     # so we can remove this else\n     pass\n\nNote that I\u0027m not really sure sub_dict_remote.get should return \u0027-1\u0027 as a default. In my understand, if the index exists in the sub dict, the value must not be None. (probably hash)","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":404,"context_line":"                        sub_dict_remote.get(remote_index, \u0027-1\u0027):"},{"line_number":405,"context_line":"                    suffixes.append(suffix)"},{"line_number":406,"context_line":"            except KeyError:"},{"line_number":407,"context_line":"                suffixes.append(suffix)"},{"line_number":408,"context_line":"        return suffixes"},{"line_number":409,"context_line":""},{"line_number":410,"context_line":"    def process_job(self, job):"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_0f8ded7a","line":407,"in_reply_to":"da9b358b_56516d0d","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":605,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":606,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":607,"context_line":"                        break"},{"line_number":608,"context_line":"                suffix_jobs.remove(job)"},{"line_number":609,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":610,"context_line":""},{"line_number":611,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_91eedfb2","line":608,"updated":"2015-03-31 09:58:16.000000000","message":"(memo) for .durable or so... not sure we need this specialization...","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":605,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":606,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":607,"context_line":"                        break"},{"line_number":608,"context_line":"                suffix_jobs.remove(job)"},{"line_number":609,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":610,"context_line":""},{"line_number":611,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_4fc1b57f","line":608,"in_reply_to":"da9b358b_91eedfb2","updated":"2015-03-31 20:55:21.000000000","message":"hopefully we can get rid of it after a refactor of this method to make the correct jobs - but the point is we don\u0027t need separate \u0027sync_only\u0027 jobs the data fragment_index and None.","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":723,"context_line":"                        self.logger.info(_(\"Ring change detected. Aborting \""},{"line_number":724,"context_line":"                                           \"current reconstruction pass.\"))"},{"line_number":725,"context_line":"                        return"},{"line_number":726,"context_line":"                    if job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":727,"context_line":"                        part_revert \u003d {}"},{"line_number":728,"context_line":"                        part_revert[\u0027path\u0027] \u003d job[\u0027path\u0027]"},{"line_number":729,"context_line":"                        part_revert[\u0027policy\u0027] \u003d job[\u0027policy\u0027]"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_11a3ef50","line":726,"updated":"2015-03-31 09:58:16.000000000","message":"Perhaps, we should make a global variables like REVERT and SYNC to decrease the possibility for typo : )","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":723,"context_line":"                        self.logger.info(_(\"Ring change detected. Aborting \""},{"line_number":724,"context_line":"                                           \"current reconstruction pass.\"))"},{"line_number":725,"context_line":"                        return"},{"line_number":726,"context_line":"                    if job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":727,"context_line":"                        part_revert \u003d {}"},{"line_number":728,"context_line":"                        part_revert[\u0027path\u0027] \u003d job[\u0027path\u0027]"},{"line_number":729,"context_line":"                        part_revert[\u0027policy\u0027] \u003d job[\u0027policy\u0027]"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_2fa1d1a3","line":726,"in_reply_to":"da9b358b_11a3ef50","updated":"2015-03-31 20:55:21.000000000","message":"Done","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"},{"line_number":428,"context_line":"        begin \u003d time.time()"},{"line_number":429,"context_line":"        # regardless of whether we are just syncing or doing a sync and"},{"line_number":430,"context_line":"        # delete, we need to run a sync job for everything that was"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_51edc523","line":427,"updated":"2015-04-01 10:30:41.000000000","message":"In this safe, and necessary, to modify the instance variable? This method is run in multiple threads, one per job, jobs have different policies...so can self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] get modified by another thread before we reach line 446?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"},{"line_number":428,"context_line":"        begin \u003d time.time()"},{"line_number":429,"context_line":"        # regardless of whether we are just syncing or doing a sync and"},{"line_number":430,"context_line":"        # delete, we need to run a sync job for everything that was"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_f6f2fd05","line":427,"in_reply_to":"baa041b7_873ee5b6","updated":"2015-04-01 18:35:38.000000000","message":"well hell then!  Given that we could have 2 different EC policies with different parms that affect more than which ring to use (like repl) its probably good to still to fix this - like quorum could be different between 2 EC policies and jack up the PUT path on the object server if policies got mixed.  I dunno, maybe not possible but worth putting some protection in when a brain cycle frees up :)","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"},{"line_number":428,"context_line":"        begin \u003d time.time()"},{"line_number":429,"context_line":"        # regardless of whether we are just syncing or doing a sync and"},{"line_number":430,"context_line":"        # delete, we need to run a sync job for everything that was"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_7a33fd14","line":427,"in_reply_to":"da9b358b_51edc523","updated":"2015-04-01 12:29:10.000000000","message":"yeah, good point.  Will fix","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1595ffd81963d3c0600455e49997eac61b264895","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"},{"line_number":428,"context_line":"        begin \u003d time.time()"},{"line_number":429,"context_line":"        # regardless of whether we are just syncing or doing a sync and"},{"line_number":430,"context_line":"        # delete, we need to run a sync job for everything that was"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_873ee5b6","line":427,"in_reply_to":"da9b358b_7a33fd14","updated":"2015-04-01 16:21:42.000000000","message":"its same in replicator :/","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":470,"context_line":"                # for regular sync we compare suffixes, for revert"},{"line_number":471,"context_line":"                # we can\u0027t as another ECrecon may have already rebuilt the"},{"line_number":472,"context_line":"                # the remote and udpated its hashes but we have the suffixes"},{"line_number":473,"context_line":"                # that need to be sync\u0027d in the job"},{"line_number":474,"context_line":"                if job[\u0027sync_type\u0027] \u003d\u003d SYNC:"},{"line_number":475,"context_line":"                    suffixes \u003d self.get_suffix_delta(job[\u0027suffixes\u0027],"},{"line_number":476,"context_line":"                                                     job[\u0027frag_index\u0027],"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_11b4ddce","line":473,"updated":"2015-04-01 10:30:41.000000000","message":"Trying to understand this... for a revert, if the remote_hash shows that a remote suffix[frag index] hash is same as ours can\u0027t we then skip the suffix? I guess we\u0027d still want to remove the handoff frag indexes locally, so is that why we have to process all suffixes regardless?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":470,"context_line":"                # for regular sync we compare suffixes, for revert"},{"line_number":471,"context_line":"                # we can\u0027t as another ECrecon may have already rebuilt the"},{"line_number":472,"context_line":"                # the remote and udpated its hashes but we have the suffixes"},{"line_number":473,"context_line":"                # that need to be sync\u0027d in the job"},{"line_number":474,"context_line":"                if job[\u0027sync_type\u0027] \u003d\u003d SYNC:"},{"line_number":475,"context_line":"                    suffixes \u003d self.get_suffix_delta(job[\u0027suffixes\u0027],"},{"line_number":476,"context_line":"                                                     job[\u0027frag_index\u0027],"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_252316a9","line":473,"in_reply_to":"da9b358b_11b4ddce","updated":"2015-04-01 12:29:10.000000000","message":"correct, the comment could be better worded (I will udpate) but yes for revert there\u0027s a race between the ECrecon doing the revert and one of the other remote node\u0027s partners doing a repair.  If the repair happens first, the ecrecon doing the revert will see matching hashes","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":470,"context_line":"                # for regular sync we compare suffixes, for revert"},{"line_number":471,"context_line":"                # we can\u0027t as another ECrecon may have already rebuilt the"},{"line_number":472,"context_line":"                # the remote and udpated its hashes but we have the suffixes"},{"line_number":473,"context_line":"                # that need to be sync\u0027d in the job"},{"line_number":474,"context_line":"                if job[\u0027sync_type\u0027] \u003d\u003d SYNC:"},{"line_number":475,"context_line":"                    suffixes \u003d self.get_suffix_delta(job[\u0027suffixes\u0027],"},{"line_number":476,"context_line":"                                                     job[\u0027frag_index\u0027],"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_f60e7de4","line":473,"in_reply_to":"da9b358b_252316a9","updated":"2015-04-01 18:35:38.000000000","message":"Done","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"                local_suff \u003d self.tpool_get_info(job[\u0027policy\u0027], job[\u0027path\u0027],"},{"line_number":489,"context_line":"                                                 recalculate\u003dsuffixes)"},{"line_number":490,"context_line":"                suffixes \u003d self.get_suffix_delta(local_suff,"},{"line_number":491,"context_line":"                                                 job[\u0027frag_index\u0027],"},{"line_number":492,"context_line":"                                                 remote_hash,"},{"line_number":493,"context_line":"                                                 node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_d150b560","line":490,"updated":"2015-04-01 10:30:41.000000000","message":"ah, so the if / else line 474 is just about which suffixes we recalculate, we do always compare with remote to determine which suffixes we sync.\n\nSO is it possible that for a REVERT, the remote got rebuilt, its suffix hash equals our local one, so we drop from the ssync run? But then when do we clean up the local handoff frag index?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"                local_suff \u003d self.tpool_get_info(job[\u0027policy\u0027], job[\u0027path\u0027],"},{"line_number":489,"context_line":"                                                 recalculate\u003dsuffixes)"},{"line_number":490,"context_line":"                suffixes \u003d self.get_suffix_delta(local_suff,"},{"line_number":491,"context_line":"                                                 job[\u0027frag_index\u0027],"},{"line_number":492,"context_line":"                                                 remote_hash,"},{"line_number":493,"context_line":"                                                 node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_d6e13900","line":490,"in_reply_to":"baa041b7_c75bcd2d","updated":"2015-04-01 18:35:38.000000000","message":"oic now... yeah I think this 2nd call to get_info needs to be skipped when we\u0027re using the job provided list.  I\u0027ll throw something in there for the next review","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"12b3297479a4f0a7cf511fa263de698ead43aaff","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"                local_suff \u003d self.tpool_get_info(job[\u0027policy\u0027], job[\u0027path\u0027],"},{"line_number":489,"context_line":"                                                 recalculate\u003dsuffixes)"},{"line_number":490,"context_line":"                suffixes \u003d self.get_suffix_delta(local_suff,"},{"line_number":491,"context_line":"                                                 job[\u0027frag_index\u0027],"},{"line_number":492,"context_line":"                                                 remote_hash,"},{"line_number":493,"context_line":"                                                 node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_f198c76c","line":490,"in_reply_to":"baa041b7_d6e13900","updated":"2015-04-01 19:05:31.000000000","message":"Done","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1595ffd81963d3c0600455e49997eac61b264895","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"                local_suff \u003d self.tpool_get_info(job[\u0027policy\u0027], job[\u0027path\u0027],"},{"line_number":489,"context_line":"                                                 recalculate\u003dsuffixes)"},{"line_number":490,"context_line":"                suffixes \u003d self.get_suffix_delta(local_suff,"},{"line_number":491,"context_line":"                                                 job[\u0027frag_index\u0027],"},{"line_number":492,"context_line":"                                                 remote_hash,"},{"line_number":493,"context_line":"                                                 node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_c75bcd2d","line":490,"in_reply_to":"da9b358b_c5534ae1","updated":"2015-04-01 16:21:42.000000000","message":"Maybe I am being slow - my \u0027worst case\u0027 was that the entire remote suffix has been rebuilt, so that suffix hash is no longer different from local and won\u0027t be included in suffixes\u003dget_suffix_delta, so not passed to ssync_sender, so we never cleanup the handoff frag indexes on local node??","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"                local_suff \u003d self.tpool_get_info(job[\u0027policy\u0027], job[\u0027path\u0027],"},{"line_number":489,"context_line":"                                                 recalculate\u003dsuffixes)"},{"line_number":490,"context_line":"                suffixes \u003d self.get_suffix_delta(local_suff,"},{"line_number":491,"context_line":"                                                 job[\u0027frag_index\u0027],"},{"line_number":492,"context_line":"                                                 remote_hash,"},{"line_number":493,"context_line":"                                                 node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_c5534ae1","line":490,"in_reply_to":"da9b358b_d150b560","updated":"2015-04-01 12:29:10.000000000","message":"you\u0027re on it.  we\u0027ll still run the ssync job in this case passing along the list of suffixes that match.  In missing_check() on the sender side there\u0027s a conditional for this case:\n\n                # this FI doesn\u0027t belong here, we\u0027ll send it if the other side\n                # wants it or not (another reconstructor may have rebuilt it\n                # already) but either way we\u0027ll delete it - well unless they\n                # asked for it but there was an error\n                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\\n                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\\n                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:\n                    self.delete_list.append((object_hash, timestamp))","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":580,"context_line":"                else:"},{"line_number":581,"context_line":"                    # if the current FI belongs here, we sync this suffix dir"},{"line_number":582,"context_line":"                    # with partners for all files matching the FI"},{"line_number":583,"context_line":"                    if partners and int(frag_index) \u003d\u003d local_node[0][\u0027index\u0027]:"},{"line_number":584,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d partners"},{"line_number":585,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d SYNC"},{"line_number":586,"context_line":"                    else:"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_db0fb42c","line":583,"updated":"2015-04-01 10:30:41.000000000","message":"if partners is empty, meaning this is a handoff node, then can the frag_index equal local node index? Maybe after a rebalance?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"12b3297479a4f0a7cf511fa263de698ead43aaff","unresolved":false,"context_lines":[{"line_number":580,"context_line":"                else:"},{"line_number":581,"context_line":"                    # if the current FI belongs here, we sync this suffix dir"},{"line_number":582,"context_line":"                    # with partners for all files matching the FI"},{"line_number":583,"context_line":"                    if partners and int(frag_index) \u003d\u003d local_node[0][\u0027index\u0027]:"},{"line_number":584,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d partners"},{"line_number":585,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d SYNC"},{"line_number":586,"context_line":"                    else:"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_916a9b19","line":583,"in_reply_to":"baa041b7_56c52959","updated":"2015-04-01 19:05:31.000000000","message":"yeah, so that could happen in which case you\u0027d drop to the else and choose to sync to whomever is now the node index at the FI that\u0027s present locally, right?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":580,"context_line":"                else:"},{"line_number":581,"context_line":"                    # if the current FI belongs here, we sync this suffix dir"},{"line_number":582,"context_line":"                    # with partners for all files matching the FI"},{"line_number":583,"context_line":"                    if partners and int(frag_index) \u003d\u003d local_node[0][\u0027index\u0027]:"},{"line_number":584,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d partners"},{"line_number":585,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d SYNC"},{"line_number":586,"context_line":"                    else:"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_56c52959","line":583,"in_reply_to":"da9b358b_450dfae6","updated":"2015-04-01 18:35:38.000000000","message":"still need to think on this one - not sure the code changes I just made make any difference","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":580,"context_line":"                else:"},{"line_number":581,"context_line":"                    # if the current FI belongs here, we sync this suffix dir"},{"line_number":582,"context_line":"                    # with partners for all files matching the FI"},{"line_number":583,"context_line":"                    if partners and int(frag_index) \u003d\u003d local_node[0][\u0027index\u0027]:"},{"line_number":584,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d partners"},{"line_number":585,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d SYNC"},{"line_number":586,"context_line":"                    else:"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_450dfae6","line":583,"in_reply_to":"da9b358b_db0fb42c","updated":"2015-04-01 12:29:10.000000000","message":"OK, didn\u0027t think of that.  Will circle back to this after reworking this routine just a bit to address Clay\u0027s comment about not having the jobs organized efficiently - right now I have one job per suffix per FI which leads to a lot of chatter, need to get to one job per FI per part","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":587,"context_line":"                        # otherwise add in everyone else we need to sync to"},{"line_number":588,"context_line":"                        # which amounts to any other nodes whose index matches"},{"line_number":589,"context_line":"                        # that of a FI still in our frag list but not already"},{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_b6bac721","line":590,"updated":"2015-04-01 10:30:41.000000000","message":"I\u0027m confused by use of \u0027frag list\u0027 in this comment and then the code below - we have one frag_index per job_info, so can there be more than one node that we revert that frag index to?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":587,"context_line":"                        # otherwise add in everyone else we need to sync to"},{"line_number":588,"context_line":"                        # which amounts to any other nodes whose index matches"},{"line_number":589,"context_line":"                        # that of a FI still in our frag list but not already"},{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_769f0d60","line":590,"in_reply_to":"da9b358b_65335eac","updated":"2015-04-01 18:35:38.000000000","message":"yeah, this is different now - ignore what was there before","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":587,"context_line":"                        # otherwise add in everyone else we need to sync to"},{"line_number":588,"context_line":"                        # which amounts to any other nodes whose index matches"},{"line_number":589,"context_line":"                        # that of a FI still in our frag list but not already"},{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_65335eac","line":590,"in_reply_to":"da9b358b_b6bac721","updated":"2015-04-01 12:29:10.000000000","message":"this is likely to change with today\u0027s rework too, will address if still relevant then","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"},{"line_number":594,"context_line":"                            # concat lists"},{"line_number":595,"context_line":"                            job_info[\u0027sync_to\u0027] +\u003d fi_node"},{"line_number":596,"context_line":"                            job_info[\u0027sync_type\u0027] \u003d REVERT"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_1bebdcb1","line":593,"updated":"2015-04-01 10:30:41.000000000","message":"fi_node is a list of ints, job_info[\u0027sync_to\u0027] is a list of ints, so fi_node can never be in job_info[\u0027sync_to\u0027]?? \n\nAnd job_info[\u0027sync_to\u0027] will be empty here always, no?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"},{"line_number":594,"context_line":"                            # concat lists"},{"line_number":595,"context_line":"                            job_info[\u0027sync_to\u0027] +\u003d fi_node"},{"line_number":596,"context_line":"                            job_info[\u0027sync_type\u0027] \u003d REVERT"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_e5e64e14","line":593,"in_reply_to":"da9b358b_1bebdcb1","updated":"2015-04-01 12:29:10.000000000","message":"same comment as above","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                        # in our sync_to list"},{"line_number":591,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":592,"context_line":"                                   int(frag_index)]"},{"line_number":593,"context_line":"                        if fi_node not in job_info[\u0027sync_to\u0027]:"},{"line_number":594,"context_line":"                            # concat lists"},{"line_number":595,"context_line":"                            job_info[\u0027sync_to\u0027] +\u003d fi_node"},{"line_number":596,"context_line":"                            job_info[\u0027sync_type\u0027] \u003d REVERT"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_f681ddfc","line":593,"in_reply_to":"da9b358b_e5e64e14","updated":"2015-04-01 18:35:38.000000000","message":"this is also gone","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":612,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":613,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":614,"context_line":"                        break"},{"line_number":615,"context_line":"                suffix_jobs.remove(job)"},{"line_number":616,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":617,"context_line":""},{"line_number":618,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_d6ab0bd5","line":615,"updated":"2015-04-01 10:30:41.000000000","message":"I\u0027m confused again - this removes the job for frag_index\u003d\u003dNone or the last job in the list if there is more than one in the list?\n\nIs the intention to remove duplicate jobs fo frag_index\u003d\u003dNone? If so, how do we get duplicates anyway? hash for each suffix is a dict keyed by frag_index so there can only be one entry with key None?","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"8e80afc19ed31a22b1090064003cb24f9218d938","unresolved":false,"context_lines":[{"line_number":612,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":613,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":614,"context_line":"                        break"},{"line_number":615,"context_line":"                suffix_jobs.remove(job)"},{"line_number":616,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":617,"context_line":""},{"line_number":618,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_2cbc986b","line":615,"in_reply_to":"baa041b7_16944135","updated":"2015-04-01 19:37:28.000000000","message":"OK, I think I get it but need to prove it to myself a bit.  The only time we need a \u0027None\u0027 job is if there are no other jobs for this part because if there are others we\u0027ll be sync\u0027ing with them either by partners for FI based but either way the durables, etc will get sync\u0027d alongside the FIs.  We only need the none jobs if there are no FI based jobs which seems odd but I guess can happen.","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":612,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":613,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":614,"context_line":"                        break"},{"line_number":615,"context_line":"                suffix_jobs.remove(job)"},{"line_number":616,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":617,"context_line":""},{"line_number":618,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_16944135","line":615,"in_reply_to":"da9b358b_65a8fe8d","updated":"2015-04-01 18:35:38.000000000","message":"this is gone now - there are no duplicates in the job list created now and that appears to be the intent of this for how it worked before","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":612,"context_line":"                    if job[\u0027frag_index\u0027] is None:"},{"line_number":613,"context_line":"                        # duplicate will be coverd by other jobs"},{"line_number":614,"context_line":"                        break"},{"line_number":615,"context_line":"                suffix_jobs.remove(job)"},{"line_number":616,"context_line":"            jobs.extend(suffix_jobs)"},{"line_number":617,"context_line":""},{"line_number":618,"context_line":"        # return a list of jobs for this part"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_65a8fe8d","line":615,"in_reply_to":"da9b358b_d6ab0bd5","updated":"2015-04-01 12:29:10.000000000","message":"beats me, Clay added this and I was going to ask the same question if this is still relevant after the changes :)","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":720,"context_line":"                for job in jobs:"},{"line_number":721,"context_line":"                    if override_devices and \\"},{"line_number":722,"context_line":"                            job[\u0027device\u0027] not in override_devices:"},{"line_number":723,"context_line":"                        continue"},{"line_number":724,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":725,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":726,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_96ffc3a9","line":723,"updated":"2015-04-01 10:30:41.000000000","message":"the replicator got changed recently so that we don\u0027t do all the work of building jobs to then throw them away when not in override_devices, we should do similar here by not calling self.build_reconstruction_jobs if device is not in override_devices. But that\u0027s not a blocker.","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":720,"context_line":"                for job in jobs:"},{"line_number":721,"context_line":"                    if override_devices and \\"},{"line_number":722,"context_line":"                            job[\u0027device\u0027] not in override_devices:"},{"line_number":723,"context_line":"                        continue"},{"line_number":724,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":725,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":726,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_65bd9eb6","line":723,"in_reply_to":"da9b358b_96ffc3a9","updated":"2015-04-01 12:29:10.000000000","message":"well, I did attempt to do that, that\u0027s why build_partitions is before the call to build jobs.  We do things in a different order here in the reconstructor though and we don\u0027t know which devices are involved until we build them so it wasn\u0027t a \"no brainer\" for the override_devices so I left it for now","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23992622fa1d36dfce262f38d6db3a289d6ac8d6","unresolved":false,"context_lines":[{"line_number":762,"context_line":"                        except OSError:"},{"line_number":763,"context_line":"                            pass"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"                    # This is an attempt to remove the partition 0 it may or"},{"line_number":766,"context_line":"                    # may not be empty and we don\u0027t care.  If not empty all"},{"line_number":767,"context_line":"                    # that means is we had some other FI\u0027s in here for some"},{"line_number":768,"context_line":"                    # reason.  If it is empty then we really were a handoff -"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_3145a16f","line":765,"updated":"2015-04-01 10:30:41.000000000","message":"typo? \u0027partition 0\u0027","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"85b26c472ca33b9d13a1e9001224cb5381aecc8d","unresolved":false,"context_lines":[{"line_number":762,"context_line":"                        except OSError:"},{"line_number":763,"context_line":"                            pass"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"                    # This is an attempt to remove the partition 0 it may or"},{"line_number":766,"context_line":"                    # may not be empty and we don\u0027t care.  If not empty all"},{"line_number":767,"context_line":"                    # that means is we had some other FI\u0027s in here for some"},{"line_number":768,"context_line":"                    # reason.  If it is empty then we really were a handoff -"}],"source_content_type":"text/x-python","patch_set":53,"id":"da9b358b_e5c9ae5a","line":765,"in_reply_to":"da9b358b_3145a16f","updated":"2015-04-01 12:29:10.000000000","message":"must be :)  thanks","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"4dbe3dae15c039d41a463515f7cd24c73614bfbd","unresolved":false,"context_lines":[{"line_number":762,"context_line":"                        except OSError:"},{"line_number":763,"context_line":"                            pass"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"                    # This is an attempt to remove the partition 0 it may or"},{"line_number":766,"context_line":"                    # may not be empty and we don\u0027t care.  If not empty all"},{"line_number":767,"context_line":"                    # that means is we had some other FI\u0027s in here for some"},{"line_number":768,"context_line":"                    # reason.  If it is empty then we really were a handoff -"}],"source_content_type":"text/x-python","patch_set":53,"id":"baa041b7_16bb21c6","line":765,"in_reply_to":"da9b358b_e5c9ae5a","updated":"2015-04-01 18:35:38.000000000","message":"Done","commit_id":"1705dd197eba83e465ae2787ad1e01fe70ad2ae5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":127,"context_line":"            \u0027user-agent\u0027: \u0027obj-reconstructor %s\u0027 % os.getpid()}"},{"line_number":128,"context_line":"        self.handoffs_first \u003d config_true_value(conf.get(\u0027handoffs_first\u0027,"},{"line_number":129,"context_line":"                                                         False))"},{"line_number":130,"context_line":"        # TODO:  handoff_delete is not implemented"},{"line_number":131,"context_line":"        self.handoff_delete \u003d config_auto_int_value("},{"line_number":132,"context_line":"            conf.get(\u0027handoff_delete\u0027, \u0027auto\u0027), 0)"},{"line_number":133,"context_line":"        self._df_router \u003d DiskFileRouter(conf, self.logger)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_6874be9b","line":130,"updated":"2015-04-02 05:51:24.000000000","message":"is still true?  let\u0027s get the option of here then - we don\u0027t have to have it - although the idea of moving parts that don\u0027t belong on this node after a rebalance *first* in order to avoid a rebuild storm seems like a pretty reasonable thing for EC to want to have ASAP.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":260,"context_line":"        # XXX with jerasure this doesn\u0027t work if we get an non-sequential list"},{"line_number":261,"context_line":"        # of fragment indexes in the payload"},{"line_number":262,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":263,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # for safty until pyeclib 1.0.7 we\u0027ll just use decode and encode"},{"line_number":266,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_1652e867","line":263,"updated":"2015-04-02 11:58:54.000000000","message":"OK, I got it. I wrote down the snippet to help this to be clear.\n\n from pyeclib.ec_iface import ECDriver\n from random import shuffle\n\n driver \u003d ECDriver(k\u003d10, m\u003d4, ec_type\u003d\u0027jerasure_rs_vand\u0027)\n frags \u003d driver.encode(\u0027 \u0027)\n\n # This results in True\n frags[0] \u003d\u003d driver.reconstruct(frags[1:], [0])[0]\n\n shuffle(frags)\n # This should also result in True because we have fragments more than k, but False :P\n frags[0] \u003d\u003d driver.reconstruct(frags[1:], [0])[0]","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c5b38c9dc00e9ecfada3b3c54ae30e115cdc7b0b","unresolved":false,"context_lines":[{"line_number":260,"context_line":"        # XXX with jerasure this doesn\u0027t work if we get an non-sequential list"},{"line_number":261,"context_line":"        # of fragment indexes in the payload"},{"line_number":262,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":263,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # for safty until pyeclib 1.0.7 we\u0027ll just use decode and encode"},{"line_number":266,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_5205fc77","line":263,"in_reply_to":"baa041b7_1652e867","updated":"2015-04-03 02:18:37.000000000","message":"I would like to revert this comment because frags[0] was shuffled too, and I\u0027m going to be in a fog because non-sequential list could be reconstructed in my env...","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cfaf4678f22f0c8793e73e688d19b0329391dc58","unresolved":false,"context_lines":[{"line_number":260,"context_line":"        # XXX with jerasure this doesn\u0027t work if we get an non-sequential list"},{"line_number":261,"context_line":"        # of fragment indexes in the payload"},{"line_number":262,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":263,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # for safty until pyeclib 1.0.7 we\u0027ll just use decode and encode"},{"line_number":266,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_cdcf317e","line":263,"in_reply_to":"baa041b7_5205fc77","updated":"2015-04-03 03:08:22.000000000","message":"I ment non-sequential as in has a hole\n\ngiven:\n\n0, 1, 2, 3, 4, 5, 6, [7, 8]\n\n0, 1, 3, 4, 5, 6, [7]\n\nCan not rebuild either 8 or 2 - with jerasure and pyeclib \u003c1.0.7\n\nhttps://gist.github.com/clayg/959a5d77353a91a4b10a","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"0b1c6006779656d1f0e303e0c0379e8b8aa86dae","unresolved":false,"context_lines":[{"line_number":260,"context_line":"        # XXX with jerasure this doesn\u0027t work if we get an non-sequential list"},{"line_number":261,"context_line":"        # of fragment indexes in the payload"},{"line_number":262,"context_line":"        # segment \u003d policy.pyeclib_driver.reconstruct("},{"line_number":263,"context_line":"        #     fragment_payload, [fragment_index])[0]"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # for safty until pyeclib 1.0.7 we\u0027ll just use decode and encode"},{"line_number":266,"context_line":"        segment \u003d policy.pyeclib_driver.decode(fragment_payload)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_df40f4a7","line":263,"in_reply_to":"baa041b7_cdcf317e","updated":"2015-04-03 05:16:22.000000000","message":"Thanks the mention and IRC talks.\n\nwith my strict test cases[1]:\n\nIt seems to have 2 condition to cause the reconstruct failure.\n- Reconstruct to a *parity* fragment\n- Available fragments doesn\u0027t have *all* data fragments. (i.e. one or more than *data* fragment(s) was not in the available fragments)\n\nHowever, this conflicts your case 2,\n\ngiven: 0,1,3,4,5,6,[7] and reconstruct 2\n\nIn my env, it worked well with\n\n driver \u003d ECDriver(k\u003d7, m\u003d2, ec_type\u003d\u0027jerasure_rs_vand\u0027)\n frags \u003d driver.encode(\" \")\n expected \u003d frags[2]\n available \u003d [frag for i, frag in enumerate(frags[:7]) if i!\u003d2]\n reconed \u003d driver.reconstruct(available, [2])\n assert reconed[0] \u003d\u003d expected\n\nWhat did I miss something?\n\nAnyways, I know it doesn\u0027t so a big matter becuase it was fixed on newest liberasurecode master :D\n\n1: https://bitbucket.org/bloodeagle40234/pyeclib/commits/ad952f5dfb66e932487c326bf012c8c1c41a7c05?at\u003dmaster","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":404,"context_line":"        return suffix_hashes"},{"line_number":405,"context_line":""},{"line_number":406,"context_line":"    def get_suffix_delta(self, local_suff, local_index,"},{"line_number":407,"context_line":"                         remote_suff, remote_index):"},{"line_number":408,"context_line":"        suffixes \u003d []"},{"line_number":409,"context_line":"        for suffix, sub_dict_local in local_suff.iteritems():"},{"line_number":410,"context_line":"            sub_dict_remote \u003d remote_suff.get(suffix, {})"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e868cebc","line":407,"updated":"2015-04-02 05:51:24.000000000","message":"docstring would be very helpful","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":404,"context_line":"        return suffix_hashes"},{"line_number":405,"context_line":""},{"line_number":406,"context_line":"    def get_suffix_delta(self, local_suff, local_index,"},{"line_number":407,"context_line":"                         remote_suff, remote_index):"},{"line_number":408,"context_line":"        suffixes \u003d []"},{"line_number":409,"context_line":"        for suffix, sub_dict_local in local_suff.iteritems():"},{"line_number":410,"context_line":"            sub_dict_remote \u003d remote_suff.get(suffix, {})"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_235fed73","line":407,"in_reply_to":"baa041b7_e868cebc","updated":"2015-04-02 11:40:58.000000000","message":"agree","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        sort of a blend of both of those operations, sometimes its"},{"line_number":422,"context_line":"        just sync\u0027ing with its partners, sometimes its just"},{"line_number":423,"context_line":"        reverting a partition and sometimes its reverting individual"},{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a8908696","line":424,"updated":"2015-04-02 05:51:24.000000000","message":"yeah but by the time you get to process_job *this* job is *either* a SYNC or REVERT - which is exactly what update and update_deleted was all about.\n\nThe difference was in reconstruction you can make this decision before examining the part\u0027s suffixes - because all replicas of a parts have the same data in them.\n\nWe just need to call build_jobs per part and *then* after examining the suffixes route to either update, or update_deleted - or for some parts - BOTH - or multiple invocations for update_deleted - for job in jobs!","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        sort of a blend of both of those operations, sometimes its"},{"line_number":422,"context_line":"        just sync\u0027ing with its partners, sometimes its just"},{"line_number":423,"context_line":"        reverting a partition and sometimes its reverting individual"},{"line_number":424,"context_line":"        objects within a suffix dir to various primiries"},{"line_number":425,"context_line":"        \"\"\""},{"line_number":426,"context_line":"        self.logger.increment(\u0027partition.delete.count.%s\u0027 % (job[\u0027device\u0027],))"},{"line_number":427,"context_line":"        self.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_83eeb989","line":424,"in_reply_to":"baa041b7_a8908696","updated":"2015-04-02 11:40:58.000000000","message":"yeah but I\u0027m not sure if you\u0027re suggesting wording clarification or just rambling :)","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":463,"context_line":"                              })"},{"line_number":464,"context_line":"                        continue"},{"line_number":465,"context_line":"                    remote_hash \u003d pickle.loads(resp.read())"},{"line_number":466,"context_line":"                    del resp"},{"line_number":467,"context_line":""},{"line_number":468,"context_line":"                # for regular sync we compare suffixes, for revert"},{"line_number":469,"context_line":"                # we can\u0027t as another ECrecon may have already rebuilt the"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_c8aceaef","line":466,"updated":"2015-04-02 05:51:24.000000000","message":"this call to the remote node is not needed if sync_type \u003d\u003d revert\n\nhere under process job revert and sync jobs should be routed to separate methods IMHO to avoid the if sync_type branches smack in the middle of a bunch of connecting to nodes and handling errors.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":463,"context_line":"                              })"},{"line_number":464,"context_line":"                        continue"},{"line_number":465,"context_line":"                    remote_hash \u003d pickle.loads(resp.read())"},{"line_number":466,"context_line":"                    del resp"},{"line_number":467,"context_line":""},{"line_number":468,"context_line":"                # for regular sync we compare suffixes, for revert"},{"line_number":469,"context_line":"                # we can\u0027t as another ECrecon may have already rebuilt the"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_23edad8b","line":466,"in_reply_to":"baa041b7_c8aceaef","updated":"2015-04-02 11:40:58.000000000","message":"not a bad idea","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":490,"context_line":"                else:"},{"line_number":491,"context_line":"                    suffixes \u003d job[\u0027hashes\u0027].keys()"},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"                ssync_sender(self, node, job, suffixes)()"},{"line_number":494,"context_line":"                with Timeout(self.http_timeout):"},{"line_number":495,"context_line":"                    conn \u003d http_connect("},{"line_number":496,"context_line":"                        node[\u0027replication_ip\u0027], node[\u0027replication_port\u0027],"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_08cf72b1","line":493,"updated":"2015-04-02 05:51:24.000000000","message":"add a comment and whitespace to highlight where we hand off to ssync to actually sync the partition.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":490,"context_line":"                else:"},{"line_number":491,"context_line":"                    suffixes \u003d job[\u0027hashes\u0027].keys()"},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"                ssync_sender(self, node, job, suffixes)()"},{"line_number":494,"context_line":"                with Timeout(self.http_timeout):"},{"line_number":495,"context_line":"                    conn \u003d http_connect("},{"line_number":496,"context_line":"                        node[\u0027replication_ip\u0027], node[\u0027replication_port\u0027],"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_0303c943","line":493,"in_reply_to":"baa041b7_08cf72b1","updated":"2015-04-02 11:40:58.000000000","message":"agree","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":503,"context_line":"            except (Exception, Timeout):"},{"line_number":504,"context_line":"                self.logger.exception("},{"line_number":505,"context_line":"                    _(\"Trying to sync suffixes with %s\") % self._full_path("},{"line_number":506,"context_line":"                        node, job[\u0027partition\u0027], \u0027\u0027, job[\u0027policy\u0027]))"},{"line_number":507,"context_line":"        self.partition_times.append(time.time() - begin)"},{"line_number":508,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":509,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e8720e7c","line":506,"updated":"2015-04-02 05:51:24.000000000","message":"Since we always make the final suffix hashes sync call this part could probably stay in process job - maybe the only difference is if you make the pre-flight sync and which suffixes you use - maybe just extra _get_suffixes_for_ssync(job)?\n\nin sync case it pre-flights and delta\u0027s in revert case it just returns our job\u0027s hashes keys?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":503,"context_line":"            except (Exception, Timeout):"},{"line_number":504,"context_line":"                self.logger.exception("},{"line_number":505,"context_line":"                    _(\"Trying to sync suffixes with %s\") % self._full_path("},{"line_number":506,"context_line":"                        node, job[\u0027partition\u0027], \u0027\u0027, job[\u0027policy\u0027]))"},{"line_number":507,"context_line":"        self.partition_times.append(time.time() - begin)"},{"line_number":508,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":509,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_83bc796d","line":506,"in_reply_to":"baa041b7_e8720e7c","updated":"2015-04-02 11:40:58.000000000","message":"thats correct","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":505,"context_line":"                    _(\"Trying to sync suffixes with %s\") % self._full_path("},{"line_number":506,"context_line":"                        node, job[\u0027partition\u0027], \u0027\u0027, job[\u0027policy\u0027]))"},{"line_number":507,"context_line":"        self.partition_times.append(time.time() - begin)"},{"line_number":508,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"    def _get_job_info(self, local_dev, part_path, partition, policy):"},{"line_number":511,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_08465223","line":508,"updated":"2015-04-02 05:51:24.000000000","message":"for sync jobs I don\u0027t think this metric makes sense.\n\nlet\u0027s pull out metrics we\u0027re not sure about and add them in Liberty on purpose.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":505,"context_line":"                    _(\"Trying to sync suffixes with %s\") % self._full_path("},{"line_number":506,"context_line":"                        node, job[\u0027partition\u0027], \u0027\u0027, job[\u0027policy\u0027]))"},{"line_number":507,"context_line":"        self.partition_times.append(time.time() - begin)"},{"line_number":508,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"    def _get_job_info(self, local_dev, part_path, partition, policy):"},{"line_number":511,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_23724de6","line":508,"in_reply_to":"baa041b7_08465223","updated":"2015-04-02 11:40:58.000000000","message":"yeah, I wasn\u0027t sure about some of these metrics - their intent wasn\u0027t always obvious to me so I just plopped them in the place that made the most sense initially and didn\u0027t really look back after it was all working","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":530,"context_line":""},{"line_number":531,"context_line":"        # dest_nodes represents all potential nodes we need to sync to"},{"line_number":532,"context_line":"        part_nodes \u003d policy.object_ring.get_part_nodes(int(partition))"},{"line_number":533,"context_line":"        local_node \u003d [n for n in part_nodes if n[\u0027id\u0027] \u003d\u003d local_id]"},{"line_number":534,"context_line":""},{"line_number":535,"context_line":"        # these are the partners we\u0027ll sync to when we\u0027re a primary"},{"line_number":536,"context_line":"        partners \u003d self._get_partners(local_id, policy.object_ring, partition)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_88eee2ea","line":533,"updated":"2015-04-02 05:51:24.000000000","message":"does local_dev not have the right index or something?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":530,"context_line":""},{"line_number":531,"context_line":"        # dest_nodes represents all potential nodes we need to sync to"},{"line_number":532,"context_line":"        part_nodes \u003d policy.object_ring.get_part_nodes(int(partition))"},{"line_number":533,"context_line":"        local_node \u003d [n for n in part_nodes if n[\u0027id\u0027] \u003d\u003d local_id]"},{"line_number":534,"context_line":""},{"line_number":535,"context_line":"        # these are the partners we\u0027ll sync to when we\u0027re a primary"},{"line_number":536,"context_line":"        partners \u003d self._get_partners(local_id, policy.object_ring, partition)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a37e5dac","line":533,"in_reply_to":"baa041b7_88eee2ea","updated":"2015-04-02 11:40:58.000000000","message":"bah, no this can be cleaned up.  too many iterations of the code","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":552,"context_line":"        # there will be one job per FI for each partition that includes"},{"line_number":553,"context_line":"        # a list of suffix dirs that need to be sync\u0027d"},{"line_number":554,"context_line":"        for suffix in hashes:"},{"line_number":555,"context_line":"            suffixes_per_fi \u003d {}"},{"line_number":556,"context_line":""},{"line_number":557,"context_line":"            for frag_index in hashes[suffix]:"},{"line_number":558,"context_line":"                job_info \u003d {}"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_42a1c093","line":555,"updated":"2015-04-02 16:06:47.000000000","message":"not studied this hard but does this want to be set here or before \"for suffix in hashes\"?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":600,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d REVERT"},{"line_number":601,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":602,"context_line":"                                   int(frag_index)]"},{"line_number":603,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d fi_node"},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"                # and now the rest of the job info needed to process..."},{"line_number":606,"context_line":"                job_info[\u0027partition\u0027] \u003d partition"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a876463c","line":603,"updated":"2015-04-02 05:51:24.000000000","message":"we don\u0027t need to recalculate this for every frag index in every suffix in the part!\n\ntranslate {suffixes: {fi: hash}} to {fi: {suffixes: hash}} then for each fi in the part build the sync_to nodes.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":600,"context_line":"                        job_info[\u0027sync_type\u0027] \u003d REVERT"},{"line_number":601,"context_line":"                        fi_node \u003d [n for n in part_nodes if n[\u0027index\u0027] \u003d\u003d"},{"line_number":602,"context_line":"                                   int(frag_index)]"},{"line_number":603,"context_line":"                        job_info[\u0027sync_to\u0027] \u003d fi_node"},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"                # and now the rest of the job info needed to process..."},{"line_number":606,"context_line":"                job_info[\u0027partition\u0027] \u003d partition"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_23492d8c","line":603,"in_reply_to":"baa041b7_a876463c","updated":"2015-04-02 11:40:58.000000000","message":"yup","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":609,"context_line":"                job_info[\u0027local_dev\u0027] \u003d local_dev"},{"line_number":610,"context_line":"                job_info[\u0027policy\u0027] \u003d policy"},{"line_number":611,"context_line":"                job_info[\u0027device\u0027] \u003d local_dev[\u0027device\u0027]"},{"line_number":612,"context_line":"                job_info[\u0027suffix_list\u0027] \u003d []"},{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_280fd6e2","line":612,"updated":"2015-04-02 05:51:24.000000000","message":"better to add this with literal dict notation when you create the dict on L558","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":609,"context_line":"                job_info[\u0027local_dev\u0027] \u003d local_dev"},{"line_number":610,"context_line":"                job_info[\u0027policy\u0027] \u003d policy"},{"line_number":611,"context_line":"                job_info[\u0027device\u0027] \u003d local_dev[\u0027device\u0027]"},{"line_number":612,"context_line":"                job_info[\u0027suffix_list\u0027] \u003d []"},{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a3553d1e","line":612,"in_reply_to":"baa041b7_280fd6e2","updated":"2015-04-02 11:40:58.000000000","message":"yeah I saw that before - just a python preference or is there a real reason?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"},{"line_number":616,"context_line":"                    job_info[\u0027local_index\u0027] \u003d \u0027-1\u0027"},{"line_number":617,"context_line":"                jobs.append(job_info)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"            # so we don\u0027t need the \u0027None\u0027 job unless there aren\u0027t any"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_68ba7ee3","line":616,"updated":"2015-04-02 05:51:24.000000000","message":"a string representing a negative integer can not be a good default here?\n\nI don\u0027t understand the difference of local_node and local_dev","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"},{"line_number":616,"context_line":"                    job_info[\u0027local_index\u0027] \u003d \u0027-1\u0027"},{"line_number":617,"context_line":"                jobs.append(job_info)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"            # so we don\u0027t need the \u0027None\u0027 job unless there aren\u0027t any"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_978828e9","line":616,"in_reply_to":"baa041b7_2b55894f","updated":"2015-04-02 16:06:47.000000000","message":"ssync_sender doesn\u0027t use local_index any more with this change: https://review.openstack.org/#/c/169052/","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"},{"line_number":616,"context_line":"                    job_info[\u0027local_index\u0027] \u003d \u0027-1\u0027"},{"line_number":617,"context_line":"                jobs.append(job_info)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"            # so we don\u0027t need the \u0027None\u0027 job unless there aren\u0027t any"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_2b55894f","line":616,"in_reply_to":"baa041b7_68ba7ee3","updated":"2015-04-02 11:58:54.000000000","message":"+1, I still wonder so. local_index seems to be used in ssyn_sender to decide to delete the local fa so maybe we can use any values unsed for the index. (e.g. None)","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                if local_node:"},{"line_number":614,"context_line":"                    job_info[\u0027local_index\u0027] \u003d local_node[0][\u0027index\u0027]"},{"line_number":615,"context_line":"                else:"},{"line_number":616,"context_line":"                    job_info[\u0027local_index\u0027] \u003d \u0027-1\u0027"},{"line_number":617,"context_line":"                jobs.append(job_info)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"            # so we don\u0027t need the \u0027None\u0027 job unless there aren\u0027t any"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e32225b9","line":616,"in_reply_to":"baa041b7_68ba7ee3","updated":"2015-04-02 11:40:58.000000000","message":"funny how an unlanded patch can have \u0027history\u0027.  I believe local_node can be replaced now","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":624,"context_line":"            if len(jobs) \u003e 1:"},{"line_number":625,"context_line":"                remove_me \u003d [j for j in jobs if j[\u0027frag_index\u0027] is None]"},{"line_number":626,"context_line":"                if remove_me:"},{"line_number":627,"context_line":"                    jobs.remove(remove_me[0])"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"            # now we know all of the suffixes affected by this FI,"},{"line_number":630,"context_line":"            # update the jobs with the suffixes they need to work on"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_88c5c25c","line":627,"updated":"2015-04-02 05:51:24.000000000","message":"me thinks better as a filter\n\n    if len(jobs) \u003e 1:\n        jobs \u003d jobs[j for j in jobs if j[\u0027frag_index\u0027] is not None]","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":624,"context_line":"            if len(jobs) \u003e 1:"},{"line_number":625,"context_line":"                remove_me \u003d [j for j in jobs if j[\u0027frag_index\u0027] is None]"},{"line_number":626,"context_line":"                if remove_me:"},{"line_number":627,"context_line":"                    jobs.remove(remove_me[0])"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"            # now we know all of the suffixes affected by this FI,"},{"line_number":630,"context_line":"            # update the jobs with the suffixes they need to work on"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_7620149c","line":627,"in_reply_to":"baa041b7_88c5c25c","updated":"2015-04-02 11:58:54.000000000","message":"+1","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":624,"context_line":"            if len(jobs) \u003e 1:"},{"line_number":625,"context_line":"                remove_me \u003d [j for j in jobs if j[\u0027frag_index\u0027] is None]"},{"line_number":626,"context_line":"                if remove_me:"},{"line_number":627,"context_line":"                    jobs.remove(remove_me[0])"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"            # now we know all of the suffixes affected by this FI,"},{"line_number":630,"context_line":"            # update the jobs with the suffixes they need to work on"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_4312f163","line":627,"in_reply_to":"baa041b7_88c5c25c","updated":"2015-04-02 11:40:58.000000000","message":"sure","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":674,"context_line":"                except OSError:"},{"line_number":675,"context_line":"                    self.logger.exception("},{"line_number":676,"context_line":"                        \u0027Unable to list partitions in %r\u0027 % obj_path)"},{"line_number":677,"context_line":"                    continue"},{"line_number":678,"context_line":"                for partition in partitions:"},{"line_number":679,"context_line":"                    part_path \u003d join(obj_path, partition)"},{"line_number":680,"context_line":"                    if not (partition.isdigit() and"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_48e21ad7","line":677,"updated":"2015-04-02 05:51:24.000000000","message":"I think here would be an interesting place to look for parts that are not supposed to be on this node for *any* fragment indexes and bump them to the front of the line to be yield out to build reconstruction jobs *first* - they should all end up being revert jobs.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":701,"context_line":"        random.shuffle(jobs)"},{"line_number":702,"context_line":"        if self.handoffs_first:"},{"line_number":703,"context_line":"            # Move the handoff revert jobs to the front of the list"},{"line_number":704,"context_line":"            jobs.sort(key\u003dlambda job: job[\u0027sync_type\u0027], reverse\u003dTrue)"},{"line_number":705,"context_line":"        self.job_count \u003d len(jobs)"},{"line_number":706,"context_line":"        return jobs"},{"line_number":707,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_28b0f6dc","line":704,"updated":"2015-04-02 05:51:24.000000000","message":"is this *really* relying on the alphabetical ordering of \u0027sync_only\u0027 and \u0027sync_revert\u0027?!\n\nCan we not do that?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":701,"context_line":"        random.shuffle(jobs)"},{"line_number":702,"context_line":"        if self.handoffs_first:"},{"line_number":703,"context_line":"            # Move the handoff revert jobs to the front of the list"},{"line_number":704,"context_line":"            jobs.sort(key\u003dlambda job: job[\u0027sync_type\u0027], reverse\u003dTrue)"},{"line_number":705,"context_line":"        self.job_count \u003d len(jobs)"},{"line_number":706,"context_line":"        return jobs"},{"line_number":707,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_51cf5ab0","line":704,"in_reply_to":"baa041b7_28b0f6dc","updated":"2015-04-02 11:58:54.000000000","message":"Not sure, but assuming the similar syntax with update_deleted, I think we can use job[\u0027local_index\u0027] is valid or not. (e.g. integer \u003e -1)\n\nIn my understanding, if local_nodes above at L533 is not empty, it means \"you are in primaries\" and then job[\u0027local_index\u0027] is assigned an index of primaries. i.e. if job[\u0027local_index\u0027] doesn\u0027t have the index, the fragment is in a handoff node.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":701,"context_line":"        random.shuffle(jobs)"},{"line_number":702,"context_line":"        if self.handoffs_first:"},{"line_number":703,"context_line":"            # Move the handoff revert jobs to the front of the list"},{"line_number":704,"context_line":"            jobs.sort(key\u003dlambda job: job[\u0027sync_type\u0027], reverse\u003dTrue)"},{"line_number":705,"context_line":"        self.job_count \u003d len(jobs)"},{"line_number":706,"context_line":"        return jobs"},{"line_number":707,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e610f350","line":704,"in_reply_to":"baa041b7_28b0f6dc","updated":"2015-04-02 11:40:58.000000000","message":"yeah, that\u0027s what its doing.  hell we sort all sorts of things alphabetically so I figured \u0027why not\u0027 but sure we can change","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":738,"context_line":"                for job in jobs:"},{"line_number":739,"context_line":"                    if override_devices and \\"},{"line_number":740,"context_line":"                            job[\u0027device\u0027] not in override_devices:"},{"line_number":741,"context_line":"                        continue"},{"line_number":742,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":743,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":744,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_c8ce2a5a","line":741,"updated":"2015-04-02 05:51:24.000000000","message":"pull this out of here - if you want o keep it move it into collect parts","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":738,"context_line":"                for job in jobs:"},{"line_number":739,"context_line":"                    if override_devices and \\"},{"line_number":740,"context_line":"                            job[\u0027device\u0027] not in override_devices:"},{"line_number":741,"context_line":"                        continue"},{"line_number":742,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":743,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":744,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_063bd7d2","line":741,"in_reply_to":"baa041b7_c8ce2a5a","updated":"2015-04-02 11:40:58.000000000","message":"I had looked at that before and it wasn\u0027t a simple \u0027just move it\u0027 but can look more closely","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":742,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":743,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":744,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"},{"line_number":745,"context_line":"                                         job[\u0027device\u0027])"},{"line_number":746,"context_line":"                        continue"},{"line_number":747,"context_line":"                    if not self.check_ring(job[\u0027policy\u0027].object_ring):"},{"line_number":748,"context_line":"                        self.logger.info(_(\"Ring change detected. Aborting \""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_28359645","line":745,"updated":"2015-04-02 05:51:24.000000000","message":"this is already in collect_parts - we do it right before the list dir - and we *just* read the hashes pkl of the damn disk in build_reconstruction jobs - the process_job is green so we\u0027ll be going through this loop pretty fast at first.\n\nThis can\u0027t be helpful to mount_check again here.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":742,"context_line":"                    dev_path \u003d join(self.devices_dir, job[\u0027device\u0027])"},{"line_number":743,"context_line":"                    if self.mount_check and not ismount(dev_path):"},{"line_number":744,"context_line":"                        self.logger.warn(_(\u0027%s is not mounted\u0027),"},{"line_number":745,"context_line":"                                         job[\u0027device\u0027])"},{"line_number":746,"context_line":"                        continue"},{"line_number":747,"context_line":"                    if not self.check_ring(job[\u0027policy\u0027].object_ring):"},{"line_number":748,"context_line":"                        self.logger.info(_(\"Ring change detected. Aborting \""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a61aeb6c","line":745,"in_reply_to":"baa041b7_28359645","updated":"2015-04-02 11:40:58.000000000","message":"yup","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":759,"context_line":"            with Timeout(self.lockup_timeout):"},{"line_number":760,"context_line":"                self.run_pool.waitall()"},{"line_number":761,"context_line":"                # now cleanup and recalc hashes.pkl before moving to"},{"line_number":762,"context_line":"                # next partition.... if we reverted something here"},{"line_number":763,"context_line":"                for revert in revert_list:"},{"line_number":764,"context_line":"                    # recalc local hashes as we deleted something"},{"line_number":765,"context_line":"                    # (either in senderor here or both)"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_085f7273","line":762,"updated":"2015-04-02 05:51:24.000000000","message":"double check those indent levels - this looks like it\u0027s after *all* the parts - not in between?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":776,"context_line":"                            os.unlink(os.path.join(revert[\u0027path\u0027],"},{"line_number":777,"context_line":"                                                   \u0027hashes.pkl\u0027))"},{"line_number":778,"context_line":"                            os.unlink(os.path.join(revert[\u0027path\u0027],"},{"line_number":779,"context_line":"                                                   \u0027.lock\u0027))"},{"line_number":780,"context_line":"                        except OSError:"},{"line_number":781,"context_line":"                            pass"},{"line_number":782,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e8db2eec","line":779,"updated":"2015-04-02 05:51:24.000000000","message":"there\u0027s actually an annoying race if you remove the .lock file.\n\nI found in https://review.openstack.org/#/c/158927/6 that it was much better to leave these files around rmtree the whole part dir on the next pass if we discovered the empty suffixes.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":776,"context_line":"                            os.unlink(os.path.join(revert[\u0027path\u0027],"},{"line_number":777,"context_line":"                                                   \u0027hashes.pkl\u0027))"},{"line_number":778,"context_line":"                            os.unlink(os.path.join(revert[\u0027path\u0027],"},{"line_number":779,"context_line":"                                                   \u0027.lock\u0027))"},{"line_number":780,"context_line":"                        except OSError:"},{"line_number":781,"context_line":"                            pass"},{"line_number":782,"context_line":""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_0642b73e","line":779,"in_reply_to":"baa041b7_e8db2eec","updated":"2015-04-02 11:40:58.000000000","message":"good call, I meant to put a TODO here to chat about it, Sam and I talked briefly about this as well","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":791,"context_line":"                        pass"},{"line_number":792,"context_line":"                    else:"},{"line_number":793,"context_line":"                        self.logger.info(_(\"Removing partition: %s\"),"},{"line_number":794,"context_line":"                                         revert[\u0027path\u0027])"},{"line_number":795,"context_line":"        except (Exception, Timeout):"},{"line_number":796,"context_line":"            self.logger.exception(_(\"Exception in top-level\""},{"line_number":797,"context_line":"                                    \"reconstruction loop\"))"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_c8518a64","line":794,"updated":"2015-04-02 05:51:24.000000000","message":"remov*ed*","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":791,"context_line":"                        pass"},{"line_number":792,"context_line":"                    else:"},{"line_number":793,"context_line":"                        self.logger.info(_(\"Removing partition: %s\"),"},{"line_number":794,"context_line":"                                         revert[\u0027path\u0027])"},{"line_number":795,"context_line":"        except (Exception, Timeout):"},{"line_number":796,"context_line":"            self.logger.exception(_(\"Exception in top-level\""},{"line_number":797,"context_line":"                                    \"reconstruction loop\"))"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_26737bb3","line":794,"in_reply_to":"baa041b7_c8518a64","updated":"2015-04-02 11:40:58.000000000","message":"done... I mean \"will do\" :)","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"}],"swift/obj/replicator.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":531,"context_line":"        jobs \u003d []"},{"line_number":532,"context_line":"        ips \u003d whataremyips()"},{"line_number":533,"context_line":"        for policy in POLICIES:"},{"line_number":534,"context_line":"            if policy.policy_type \u003d\u003d REPL_POLICY:"},{"line_number":535,"context_line":"                if (override_policies is not None and"},{"line_number":536,"context_line":"                        str(policy.idx) not in override_policies):"},{"line_number":537,"context_line":"                    continue"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_88f94250","line":534,"updated":"2015-04-02 05:51:24.000000000","message":"heh, as far as indention bugs go this one is pretty good - no tests to catch it?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"96e62065c11d573b8a4a9adbff631d7d29da645f","unresolved":false,"context_lines":[{"line_number":531,"context_line":"        jobs \u003d []"},{"line_number":532,"context_line":"        ips \u003d whataremyips()"},{"line_number":533,"context_line":"        for policy in POLICIES:"},{"line_number":534,"context_line":"            if policy.policy_type \u003d\u003d REPL_POLICY:"},{"line_number":535,"context_line":"                if (override_policies is not None and"},{"line_number":536,"context_line":"                        str(policy.idx) not in override_policies):"},{"line_number":537,"context_line":"                    continue"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e6ac73ef","line":534,"in_reply_to":"baa041b7_88f94250","updated":"2015-04-02 11:40:58.000000000","message":"we might catch it now, would have to reintroduce it and re-run which is probably worth doing","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"}],"swift/obj/ssync_receiver.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ad87a911fc87a2f3030e2a01385ead20b9225cf2","unresolved":false,"context_lines":[{"line_number":226,"context_line":"                line \u003d self.fp.readline(self.app.network_chunk_size)"},{"line_number":227,"context_line":"            if not line or line.strip() \u003d\u003d \u0027:MISSING_CHECK: END\u0027:"},{"line_number":228,"context_line":"                break"},{"line_number":229,"context_line":"            object_hash, timestamp \u003d [urllib.unquote(v) for v in line.split()]"},{"line_number":230,"context_line":"            want \u003d False"},{"line_number":231,"context_line":"            try:"},{"line_number":232,"context_line":"                df \u003d self.app._diskfile_mgr.get_diskfile_from_hash("}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_f763a7ae","side":"PARENT","line":229,"updated":"2015-01-06 01:42:10.000000000","message":"there\u0027s a bunch of places where we call timestamp-fragment_index the \"timestamp\" - which is annoying to reason about and probably error prone.  In this particular case we don\u0027t even use fragment_index when we decide if we want this timestamp or not - but we probably... should?  Like if you\u0027re trying to send me ts1-3 and I have ts1-4 it\u0027s not really fair for me to say \"nah I got that\".\n\nI think we\u0027re going to have to figure out how to make diskfiles that can have multiple fragment_indexes behind them?","commit_id":"4feca7f0991a2fe7aa248b0e9540a030a8d1cad7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a44b824eee81d98834076ce5cdb5f048cde7311f","unresolved":false,"context_lines":[{"line_number":226,"context_line":"                line \u003d self.fp.readline(self.app.network_chunk_size)"},{"line_number":227,"context_line":"            if not line or line.strip() \u003d\u003d \u0027:MISSING_CHECK: END\u0027:"},{"line_number":228,"context_line":"                break"},{"line_number":229,"context_line":"            object_hash, timestamp \u003d [urllib.unquote(v) for v in line.split()]"},{"line_number":230,"context_line":"            want \u003d False"},{"line_number":231,"context_line":"            try:"},{"line_number":232,"context_line":"                df \u003d self.app._diskfile_mgr.get_diskfile_from_hash("}],"source_content_type":"text/x-python","patch_set":13,"id":"3a961159_3b166ef3","side":"PARENT","line":229,"in_reply_to":"3a961159_f763a7ae","updated":"2015-01-19 23:59:55.000000000","message":"yeah, we should look at all this again after the next patch set (from now, not from set 13) as having multiple FI\u0027s in one obj hash dir is in its fledgling state there","commit_id":"4feca7f0991a2fe7aa248b0e9540a030a8d1cad7"}],"swift/obj/ssync_sender.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be6df71f87fccf467e01c96e4d4ab5754214880f","unresolved":false,"context_lines":[{"line_number":274,"context_line":"                                                        self.policy,"},{"line_number":275,"context_line":"                                                        df.get_metadata())"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"                self.send_put(url_path, df, extra_headers)"},{"line_number":278,"context_line":"        with exceptions.MessageTimeout("},{"line_number":279,"context_line":"                self.daemon.node_timeout, \u0027updates end\u0027):"},{"line_number":280,"context_line":"            msg \u003d \u0027:UPDATES: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_da0fec86","line":277,"updated":"2015-02-09 20:06:56.000000000","message":"build the list of names to delete and append to it here, then below you can just remove those names w/o having to open the df again.","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be6df71f87fccf467e01c96e4d4ab5754214880f","unresolved":false,"context_lines":[{"line_number":351,"context_line":"            if key not in (\u0027name\u0027, \u0027Content-Length\u0027):"},{"line_number":352,"context_line":"                msg.append(\u0027%s: %s\u0027 % (key, value))"},{"line_number":353,"context_line":"        for key, value in extra_headers:"},{"line_number":354,"context_line":"            msg.append(\u0027%s: %s\u0027 % (key, value))"},{"line_number":355,"context_line":"        msg \u003d \u0027\\r\\n\u0027.join(msg) + \u0027\\r\\n\\r\\n\u0027"},{"line_number":356,"context_line":"        with exceptions.MessageTimeout(self.daemon.node_timeout, \u0027send_put\u0027):"},{"line_number":357,"context_line":"            self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(msg), msg))"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_3aad6889","line":354,"updated":"2015-02-09 20:06:56.000000000","message":"the fragment archive index \"extra_headers\" are already getting added above because the DiskFileLikeThing put them in the df.get_metadata()","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"6025d520e5e61199ff785426ab94f2a7832a38b7","unresolved":false,"context_lines":[{"line_number":351,"context_line":"            if key not in (\u0027name\u0027, \u0027Content-Length\u0027):"},{"line_number":352,"context_line":"                msg.append(\u0027%s: %s\u0027 % (key, value))"},{"line_number":353,"context_line":"        for key, value in extra_headers:"},{"line_number":354,"context_line":"            msg.append(\u0027%s: %s\u0027 % (key, value))"},{"line_number":355,"context_line":"        msg \u003d \u0027\\r\\n\u0027.join(msg) + \u0027\\r\\n\\r\\n\u0027"},{"line_number":356,"context_line":"        with exceptions.MessageTimeout(self.daemon.node_timeout, \u0027send_put\u0027):"},{"line_number":357,"context_line":"            self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(msg), msg))"}],"source_content_type":"text/x-python","patch_set":18,"id":"da86d52c_1cb06942","line":354,"in_reply_to":"da86d52c_3aad6889","updated":"2015-02-10 20:44:37.000000000","message":"yeah, had to rework this little section a bit to make it work correctly but you\u0027re right I was ending up with the header in the msg twice","commit_id":"36936e452076dd5e59344e2f4d9c88e7ba933bff"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"43730faed8eb732280359af953c8fb89b134b750","unresolved":false,"context_lines":[{"line_number":362,"context_line":"                    delete_path \u003d \\"},{"line_number":363,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":364,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":365,"context_line":"                        os.unlink(delete_path)"},{"line_number":366,"context_line":""},{"line_number":367,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":368,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":33,"id":"9a80dd14_f099926d","line":365,"updated":"2015-03-20 20:12:49.000000000","message":"this reminded me of https://bugs.launchpad.net/swift/+bug/1408061 and https://review.openstack.org/#/c/158927/","commit_id":"c6f121c3f788f91b3fa76261bcb1d7cbbdb7c4de"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":49,"context_line":"        self.delete_list \u003d []"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"    @property"},{"line_number":52,"context_line":"    def frag_index(self):"},{"line_number":53,"context_line":"        return self.job.get(\u0027frag_index\u0027)"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    def __call__(self):"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_bae43c3b","line":52,"updated":"2015-03-30 19:25:55.000000000","message":"any reason why there is a property for frag_index? - throughout the module the pattern is to call self.job[\u003ckey\u003e} for other job parameters","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":220,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":221,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":222,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":223,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":224,"context_line":"        with exceptions.MessageTimeout("},{"line_number":225,"context_line":"                self.daemon.node_timeout, \u0027missing_check end\u0027):"},{"line_number":226,"context_line":"            msg \u003d \u0027:MISSING_CHECK: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_150d3b3e","line":223,"updated":"2015-03-30 19:25:55.000000000","message":"i think this could be done at line 289 where most of the same conditions are applied, so less lines of change and all in one place. And delete_list could then be scoped just to updates method.","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":283,"context_line":"                # If this is an EC policy, we need to reconstruct"},{"line_number":284,"context_line":"                # the right fragment archive before sending it over"},{"line_number":285,"context_line":"                # unless we\u0027re sync\u0027ing because of an update_delete()"},{"line_number":286,"context_line":"                # job in which case we just send it over \"as is\""},{"line_number":287,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":288,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_dac82077","line":286,"updated":"2015-03-30 19:25:55.000000000","message":"nit: Split and move comment after each of the if self.job[\u0027sync_type\u0027], elif self.job[\u0027sync_type\u0027] options so its clearer. \n\nNot clear what an update_delete() job is?","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":284,"context_line":"                # the right fragment archive before sending it over"},{"line_number":285,"context_line":"                # unless we\u0027re sync\u0027ing because of an update_delete()"},{"line_number":286,"context_line":"                # job in which case we just send it over \"as is\""},{"line_number":287,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":288,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":290,"context_line":"                            self.job[\u0027frag_index\u0027]"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_9a8f3823","line":287,"updated":"2015-03-30 19:25:55.000000000","message":"# for an EC policy there are two types of updates...","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":285,"context_line":"                # unless we\u0027re sync\u0027ing because of an update_delete()"},{"line_number":286,"context_line":"                # job in which case we just send it over \"as is\""},{"line_number":287,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":288,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":290,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":291,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_faa08492","line":288,"updated":"2015-03-30 19:25:55.000000000","message":"# if we have the right fragment for the receiver then just send it","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":287,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":288,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":290,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":291,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":292,"context_line":"                    elif self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_only\u0027:"},{"line_number":293,"context_line":"                        # create a new df-like thing for the rebuilt object"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_7a17d4a3","line":290,"updated":"2015-03-30 19:25:55.000000000","message":"Won\u0027t \u0027X-Object-Sysmeta-Ec-Archive-Index\u0027 be in metadata for the frag index .data file, so when we open the df with that frag index it should have \u0027X-Object-Sysmeta-Ec-Archive-Index\u0027 already in metadata, therefore no need to add it here.\n\n(Its like how the reconstrcuted df (the next case) has X-Object-Sysmeta-Ec-Archive-Index added to its df-like-thing metadata)\n\nAlso, everywhere in obj server we refer to \u0027fragment index\u0027 but the header is an \u0027archive index\u0027 - should we harmonize?","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":288,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":290,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":291,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":292,"context_line":"                    elif self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_only\u0027:"},{"line_number":293,"context_line":"                        # create a new df-like thing for the rebuilt object"},{"line_number":294,"context_line":"                        df \u003d self.daemon.reconstruct_fa(self.job, self.node,"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_1adc48e3","line":291,"updated":"2015-03-30 19:25:55.000000000","message":"not required?","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":289,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":290,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":291,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":292,"context_line":"                    elif self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_only\u0027:"},{"line_number":293,"context_line":"                        # create a new df-like thing for the rebuilt object"},{"line_number":294,"context_line":"                        df \u003d self.daemon.reconstruct_fa(self.job, self.node,"},{"line_number":295,"context_line":"                                                        self.job[\u0027policy\u0027],"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_7ae234d7","line":292,"updated":"2015-03-30 19:25:55.000000000","message":"# otherwise we need to reconstruct the right fragment before sending","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":295,"context_line":"                                                        self.job[\u0027policy\u0027],"},{"line_number":296,"context_line":"                                                        df.get_metadata())"},{"line_number":297,"context_line":"                        if df is None:"},{"line_number":298,"context_line":"                            # TODO, broken pipe maybe from here?"},{"line_number":299,"context_line":"                            with exceptions.MessageTimeout("},{"line_number":300,"context_line":"                                    self.daemon.node_timeout, \u0027updates end\u0027):"},{"line_number":301,"context_line":"                                msg \u003d \u0027:UPDATES: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_557ce371","line":298,"updated":"2015-03-30 19:25:55.000000000","message":"hmmm, why bail out just because this object can\u0027t be reconstructed? looks like ssync receiver doesn\u0027t check that all the hashes returned in missing_checks phase actually get sent in updates phase, so just ignore this df and move on down the send_list, no? Otherwise can\u0027t we end up with a pathological case of a non-reconstructable file that *always* prevents other files being sync\u0027d.","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":336,"context_line":"        # so we\u0027ll delete reverted objects here and clean up the"},{"line_number":337,"context_line":"        # part dir, if empty, in the reconstructor, note we delete"},{"line_number":338,"context_line":"        # the FI of the file we sent to the target..."},{"line_number":339,"context_line":"        if self.job and self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":340,"context_line":"            if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":341,"context_line":"                for object_hash, timestamp in self.delete_list:"},{"line_number":342,"context_line":"                    try:"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_75e247d7","line":339,"updated":"2015-03-30 19:25:55.000000000","message":"there is no check for self.job before this point so \"if self.job\" can\u0027t be necessary here","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":349,"context_line":"                        continue"},{"line_number":350,"context_line":"                    delete_path \u003d \\"},{"line_number":351,"context_line":"                        os.path.join(df._datadir,"},{"line_number":352,"context_line":"                                     self.job[\u0027policy\u0027].ts_to_fname"},{"line_number":353,"context_line":"                                     (timestamp,"},{"line_number":354,"context_line":"                                      frag_index\u003dself.node[\u0027index\u0027])"},{"line_number":355,"context_line":"                                     + \u0027.data\u0027)"}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_30aa8dd5","line":352,"updated":"2015-03-30 19:25:55.000000000","message":"ts_to_fname doesn\u0027t exist in policy classes and is renamed in diskfile.\n\nwe ought to have a method on diskfile class to get the data file name maybe? In fact, we ought to have a kwarg on delete() method on diskfile so we can call df.delete(frag_index\u003dN) and it will do \"the right thing\".","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"128d4b1a65a30986bd1a5e1b4d7713d4e432afe6","unresolved":false,"context_lines":[{"line_number":357,"context_line":"                    delete_path \u003d \\"},{"line_number":358,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":359,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":360,"context_line":"                        os.unlink(delete_path)"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":363,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":45,"id":"da9b358b_b000ddab","line":360,"updated":"2015-03-30 19:25:55.000000000","message":"so if we are deleting the durable file, does a sync_revert job mean there is no frag index we want to keep on this node? Or do we want to only delete the durable if there is no local frag archive?","commit_id":"a53bcdc0eda90644a632e50e982e1575099c6e7b"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":357,"context_line":"                    delete_path \u003d \\"},{"line_number":358,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":359,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":360,"context_line":"                        os.unlink(delete_path)"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":363,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_0c451216","line":360,"updated":"2015-03-31 09:58:16.000000000","message":"Not sure but this staff from L335 seems the role of the reconstructor because replicator does similar staff in update_deleted when the cleanup objects are in the handoff node.\n\nI know current ssync has a possibility to return only the object hash like as global replication, but almost of the information to construct the diskfile (e.g. frag_index, policy) seems to be given from reconstructor.\n\nFor now, I\u0027m thinking for improvement this code, so tentatively  this is not blocker to me.","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":357,"context_line":"                    delete_path \u003d \\"},{"line_number":358,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":359,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":360,"context_line":"                        os.unlink(delete_path)"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":363,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_efde4910","line":360,"in_reply_to":"da9b358b_0c451216","updated":"2015-03-31 20:55:21.000000000","message":"+1 this sticks out being added here - we should think carefully about leaking this responsibility into ssync - i\u0027m also trying to think about it and agree the global replication sync sets have a good chance of working out.","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"df76fb49a382e0aa32ab3953dc0c329bbf3ab5fd","unresolved":false,"context_lines":[{"line_number":357,"context_line":"                    delete_path \u003d \\"},{"line_number":358,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":359,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":360,"context_line":"                        os.unlink(delete_path)"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":363,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":49,"id":"baa041b7_487e9ae5","line":360,"in_reply_to":"da9b358b_efde4910","updated":"2015-04-02 05:16:45.000000000","message":"Ok, I\u0027ll continue to review later patch set with keeping this in my head. Thanks!","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":325,"context_line":"        \"\"\""},{"line_number":326,"context_line":"        Sends a PUT subrequest for the url_path using the source df"},{"line_number":327,"context_line":"        (DiskFile) and content_length."},{"line_number":328,"context_line":"        \"\"\""},{"line_number":329,"context_line":"        msg \u003d [\u0027PUT \u0027 + url_path, \u0027Content-Length: \u0027 + str(df.content_length)]"},{"line_number":330,"context_line":"        # Sorted to make it easier to test."},{"line_number":331,"context_line":"        for key, value in sorted(df.get_metadata().iteritems()):"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_63d3bf39","side":"PARENT","line":328,"updated":"2015-04-02 05:51:24.000000000","message":"yup right here if EC get a different diskfile than the one passed in.  seems simple enough?  maybe tests hate it or something?","commit_id":"3388acdd52a5167b65c5a7c24b33f4a29884b570"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":220,"context_line":"                # already) but either way we\u0027ll delete it - well unless they"},{"line_number":221,"context_line":"                # asked for it but there was an error"},{"line_number":222,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":223,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":224,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":225,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":226,"context_line":"        with exceptions.MessageTimeout("}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_31b17690","line":223,"updated":"2015-04-02 11:58:54.000000000","message":"please use REVERT global variable in the reconstructor. 2 more same and 1 similar (i.e. sync_only) things found below.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":220,"context_line":"                # already) but either way we\u0027ll delete it - well unless they"},{"line_number":221,"context_line":"                # asked for it but there was an error"},{"line_number":222,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":223,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":224,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":225,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":226,"context_line":"        with exceptions.MessageTimeout("}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_62b04444","line":223,"in_reply_to":"baa041b7_31b17690","updated":"2015-04-02 16:06:47.000000000","message":"+1 noted","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":223,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":224,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":225,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":226,"context_line":"        with exceptions.MessageTimeout("},{"line_number":227,"context_line":"                self.daemon.node_timeout, \u0027missing_check end\u0027):"},{"line_number":228,"context_line":"            msg \u003d \u0027:MISSING_CHECK: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_481d7a72","line":225,"updated":"2015-04-02 05:51:24.000000000","message":"isn\u0027t this just available_set - maybe the difference is the timestamp - better to update that set than to duplicate the information?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":223,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":224,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":225,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":226,"context_line":"        with exceptions.MessageTimeout("},{"line_number":227,"context_line":"                self.daemon.node_timeout, \u0027missing_check end\u0027):"},{"line_number":228,"context_line":"            msg \u003d \u0027:MISSING_CHECK: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_716dbe03","line":225,"in_reply_to":"baa041b7_481d7a72","updated":"2015-04-02 11:58:54.000000000","message":"+1","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY and \\"},{"line_number":223,"context_line":"                        self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027 and \\"},{"line_number":224,"context_line":"                        self.job[\u0027local_index\u0027] !\u003d self.node[\u0027index\u0027]:"},{"line_number":225,"context_line":"                    self.delete_list.append((object_hash, timestamp))"},{"line_number":226,"context_line":"        with exceptions.MessageTimeout("},{"line_number":227,"context_line":"                self.daemon.node_timeout, \u0027missing_check end\u0027):"},{"line_number":228,"context_line":"            msg \u003d \u0027:MISSING_CHECK: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_825c6826","line":225,"in_reply_to":"baa041b7_481d7a72","updated":"2015-04-02 16:06:47.000000000","message":"I spent a while looking into that here https://review.openstack.org/#/c/169052/ but ran out of time - we should do it and change return type from ssync_sender but I didn\u0027t want to go *there* right now in case someone out there is actually using it.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":289,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":290,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":291,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":292,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":293,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":294,"context_line":"                    elif self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_only\u0027:"},{"line_number":295,"context_line":"                        # create a new df-like thing for the rebuilt object"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_63d05f54","line":292,"updated":"2015-04-02 05:51:24.000000000","message":"our StreamRebuildingDiskFile does almost exactly this same thing - is the objects sys meta really not going to be included in the PUT otherwise?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":289,"context_line":"                if self.job[\u0027policy\u0027].policy_type \u003d\u003d EC_POLICY:"},{"line_number":290,"context_line":"                    if self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_revert\u0027:"},{"line_number":291,"context_line":"                        extra_headers[\u0027X-Object-Sysmeta-Ec-Archive-Index\u0027] \u003d \\"},{"line_number":292,"context_line":"                            self.job[\u0027frag_index\u0027]"},{"line_number":293,"context_line":"                        #extra_headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":294,"context_line":"                    elif self.job[\u0027sync_type\u0027] \u003d\u003d \u0027sync_only\u0027:"},{"line_number":295,"context_line":"                        # create a new df-like thing for the rebuilt object"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_42c2e086","line":292,"in_reply_to":"baa041b7_63d05f54","updated":"2015-04-02 16:06:47.000000000","message":"+1 See TODO in https://review.openstack.org/#/c/169052/ I don\u0027t think we need to do this but I keep fretting about how a disk file has two ways to get a frag_index (constructor and metadata) so in theory could have a frag index but not have \u0027X-Object-Sysmeta-Ec-Archive-Index\u0027, but in practice idk :/","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":304,"context_line":"                                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 %"},{"line_number":305,"context_line":"                                                     (len(msg), msg))"},{"line_number":306,"context_line":"                            return"},{"line_number":307,"context_line":"                self.send_put(url_path, df, extra_headers)"},{"line_number":308,"context_line":"        with exceptions.MessageTimeout("},{"line_number":309,"context_line":"                self.daemon.node_timeout, \u0027updates end\u0027):"},{"line_number":310,"context_line":"            msg \u003d \u0027:UPDATES: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_a31ae715","line":307,"updated":"2015-04-02 05:51:24.000000000","message":"I think I\u0027d rather see the ec specific logic to rebuild the fa at the top of the rather small send_put method - instead of nosily slipped into the else block of the already complicated exception handling code here in the the send list iteration of update.\n\nside benift you don\u0027t need to pass in extra_headers (assuming you really need to pass in extra_headers)","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":304,"context_line":"                                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 %"},{"line_number":305,"context_line":"                                                     (len(msg), msg))"},{"line_number":306,"context_line":"                            return"},{"line_number":307,"context_line":"                self.send_put(url_path, df, extra_headers)"},{"line_number":308,"context_line":"        with exceptions.MessageTimeout("},{"line_number":309,"context_line":"                self.daemon.node_timeout, \u0027updates end\u0027):"},{"line_number":310,"context_line":"            msg \u003d \u0027:UPDATES: END\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_825d88eb","line":307,"in_reply_to":"baa041b7_a31ae715","updated":"2015-04-02 16:06:47.000000000","message":"a little cleaner in https://review.openstack.org/#/c/169052/","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":358,"context_line":"                    delete_path \u003d \\"},{"line_number":359,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":360,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":361,"context_line":"                        os.unlink(delete_path)"},{"line_number":362,"context_line":""},{"line_number":363,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":364,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e396af93","line":361,"updated":"2015-04-02 05:51:24.000000000","message":"I don\u0027t think it\u0027s a good idea to do this here while we still have a connection open with the other node - maybe after disconnect - or even better - just hand it over to the reconstructor and let him do the right thing - same as we\u0027ve done/are doing with global replication by passing out the available_set (which should be a dict that maps hashes to timestamps I\u0027m seeing now)","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f72c652223ca9e2440146fb95547022afee4746b","unresolved":false,"context_lines":[{"line_number":358,"context_line":"                    delete_path \u003d \\"},{"line_number":359,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":360,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":361,"context_line":"                        os.unlink(delete_path)"},{"line_number":362,"context_line":""},{"line_number":363,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":364,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_6221644d","line":361,"in_reply_to":"baa041b7_e396af93","updated":"2015-04-02 16:06:47.000000000","message":"+1 I want to make ssync_sender return a delete_list and have the recosntructor do its own cleanup, better separation.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":358,"context_line":"                    delete_path \u003d \\"},{"line_number":359,"context_line":"                        os.path.join(df._datadir, timestamp + \u0027.durable\u0027)"},{"line_number":360,"context_line":"                    if os.path.isfile(delete_path):"},{"line_number":361,"context_line":"                        os.unlink(delete_path)"},{"line_number":362,"context_line":""},{"line_number":363,"context_line":"    def send_delete(self, url_path, timestamp):"},{"line_number":364,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_63ee1524","line":361,"in_reply_to":"baa041b7_e396af93","updated":"2015-04-02 11:58:54.000000000","message":"+1, thanks to consider my commet :D","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"}],"test/unit/obj/test_reconstructor.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"6903e032d4d01bd0ec6971f64417fae7d10d6a56","unresolved":false,"context_lines":[{"line_number":848,"context_line":"        # into build_jobs and then spawns it\u0027s safe to do the remove_files"},{"line_number":849,"context_line":"        # without making reconstructor startup slow"},{"line_number":850,"context_line":"        for part_info in self.reconstructor.collect_parts():"},{"line_number":851,"context_line":"            self.assertFalse(os.path.exists(pol_1_part_1_path))"},{"line_number":852,"context_line":""},{"line_number":853,"context_line":"    def test_delete_partition(self):"},{"line_number":854,"context_line":"        # part 2 is predefined to have all revert jobs"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_2cac569f","line":851,"updated":"2015-03-31 09:58:16.000000000","message":"I didn\u0027t get the reason yet but this assertion goes to fail in my environment :(","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"902b8e720690349e139b9f27bed4e586c520bbfe","unresolved":false,"context_lines":[{"line_number":848,"context_line":"        # into build_jobs and then spawns it\u0027s safe to do the remove_files"},{"line_number":849,"context_line":"        # without making reconstructor startup slow"},{"line_number":850,"context_line":"        for part_info in self.reconstructor.collect_parts():"},{"line_number":851,"context_line":"            self.assertFalse(os.path.exists(pol_1_part_1_path))"},{"line_number":852,"context_line":""},{"line_number":853,"context_line":"    def test_delete_partition(self):"},{"line_number":854,"context_line":"        # part 2 is predefined to have all revert jobs"}],"source_content_type":"text/x-python","patch_set":49,"id":"da9b358b_2ffe7137","line":851,"in_reply_to":"da9b358b_2cac569f","updated":"2015-03-31 20:55:21.000000000","message":"hrmm... ok let me know!","commit_id":"8f5981f16bd0843c51f2143db9401c25d151568c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":241,"context_line":"        for policy in [p for p in POLICIES if p.policy_type \u003d\u003d EC_POLICY]:"},{"line_number":242,"context_line":"            self.ec_policy \u003d policy"},{"line_number":243,"context_line":"            self.ec_obj_ring \u003d \\"},{"line_number":244,"context_line":"                self.reconstructor.get_object_ring(self.ec_policy.idx)"},{"line_number":245,"context_line":"            data_dir \u003d diskfile.get_data_dir(self.ec_policy)"},{"line_number":246,"context_line":"            for local_dev in [dev for dev in self.ec_obj_ring.devs"},{"line_number":247,"context_line":"                              if dev and dev[\u0027replication_ip\u0027] in ips and"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_83d35938","line":244,"updated":"2015-04-02 11:58:54.000000000","message":"Let\u0027s use int(policy) syntax :D","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":253,"context_line":"                self.ec_obj_path \u003d os.path.join(dev_path, data_dir)"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"                # create a bunch of FA\u0027s to test"},{"line_number":256,"context_line":"                t \u003d 1421181937.70054  # time.time()"},{"line_number":257,"context_line":"                _create_frag_archives(self.ec_policy, self.ec_obj_path,"},{"line_number":258,"context_line":"                                      self.ec_local_dev[\u0027id\u0027], 0)"},{"line_number":259,"context_line":"                _create_frag_archives(self.ec_policy, self.ec_obj_path,"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_830599c3","line":256,"updated":"2015-04-02 11:58:54.000000000","message":"What do you use this \u0027t\u0027 for?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":286,"context_line":""},{"line_number":287,"context_line":"    def debug_wtf(self):"},{"line_number":288,"context_line":"        # won\u0027t include this in the final, just handy reminder of where"},{"line_number":289,"context_line":"        # things are..."},{"line_number":290,"context_line":"        for pol in [p for p in POLICIES if p.policy_type \u003d\u003d EC_POLICY]:"},{"line_number":291,"context_line":"            obj_ring \u003d self.reconstructor.get_object_ring(pol.idx)"},{"line_number":292,"context_line":"            for part_num in self.part_nums:"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_881a626d","line":289,"updated":"2015-04-02 05:51:24.000000000","message":"we\u0027re getting pretty close to final - but if you wanna leave this I can take it out in ec_review","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"dc4a04481cdd13aaa41821f38571426c83542d26","unresolved":false,"context_lines":[{"line_number":1572,"context_line":"        self.assertEqual(len(ssync_calls), 1)"},{"line_number":1573,"context_line":"        call \u003d ssync_calls[0]"},{"line_number":1574,"context_line":"        self.assertEqual(call[\u0027node\u0027], sync_to[0])"},{"line_number":1575,"context_line":"        self.assertEqual(set(call[\u0027hashes\u0027]), set([\u0027123\u0027, \u0027abc\u0027]))"},{"line_number":1576,"context_line":""},{"line_number":1577,"context_line":"    def test_reconstruct_fa_no_errors(self):"},{"line_number":1578,"context_line":"        job \u003d {"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_0673d781","line":1575,"updated":"2015-04-02 11:58:54.000000000","message":"Those process_job tests seems good to me but I\u0027d like to care about \u0027local_index\u0027 value on the job.","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":1766,"context_line":"        pass"},{"line_number":1767,"context_line":""},{"line_number":1768,"context_line":"    def test_reconstruct_revert_paths(self):"},{"line_number":1769,"context_line":"        pass"},{"line_number":1770,"context_line":""},{"line_number":1771,"context_line":""},{"line_number":1772,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_c8374ae2","line":1769,"updated":"2015-04-02 05:51:24.000000000","message":"ok, at least some of these are covered in the GlobalSetup tests - which seem to be sticking around - so let\u0027s get rid of these stubs?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7fb23eb96bf9a0a1f9e043c2ae945e4b1c4afd23","unresolved":false,"context_lines":[{"line_number":5099,"context_line":"        footer_meta \u003d {"},{"line_number":5100,"context_line":"            \"X-Object-Sysmeta-Ec-Archive-Index\": \"7\","},{"line_number":5101,"context_line":"            \"Etag\": md5(test_data).hexdigest(),"},{"line_number":5102,"context_line":"        }"},{"line_number":5103,"context_line":"        footer_json \u003d json.dumps(footer_meta)"},{"line_number":5104,"context_line":"        footer_meta_cksum \u003d md5(footer_json).hexdigest()"},{"line_number":5105,"context_line":"        test_doc \u003d \"\\r\\n\".join(("}],"source_content_type":"text/x-python","patch_set":56,"id":"baa041b7_e85e8e44","line":5102,"updated":"2015-04-02 05:51:24.000000000","message":"this looks like me - but I can\u0027t fathom why I wrote it here in the reconstructor change?","commit_id":"a43e37a5d202909383692c8915ac0942825a5767"}]}