)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":11,"context_line":""},{"line_number":12,"context_line":"But the reason I make this as separate one is to keep this patch clean"},{"line_number":13,"context_line":"to know what\u0027s changed from the parent to reviewers so that if it could"},{"line_number":14,"context_line":"be okay to get this into the parent, I will squash this up to there."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Originally, ec duplicaiton is designed to make duplicated copy at"},{"line_number":17,"context_line":"chunk_transformer function for each node. However, it can cause doubled"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"ba2be162_9cec5594","line":14,"updated":"2017-03-06 18:50:28.000000000","message":"the first two paragraphs are obsolete now that the parent has merged.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":11,"context_line":""},{"line_number":12,"context_line":"But the reason I make this as separate one is to keep this patch clean"},{"line_number":13,"context_line":"to know what\u0027s changed from the parent to reviewers so that if it could"},{"line_number":14,"context_line":"be okay to get this into the parent, I will squash this up to there."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Originally, ec duplicaiton is designed to make duplicated copy at"},{"line_number":17,"context_line":"chunk_transformer function for each node. However, it can cause doubled"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"9a30ddce_ce45eeb3","line":14,"in_reply_to":"ba2be162_9cec5594","updated":"2017-03-08 09:21:07.000000000","message":"Done","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":13,"context_line":"to know what\u0027s changed from the parent to reviewers so that if it could"},{"line_number":14,"context_line":"be okay to get this into the parent, I will squash this up to there."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Originally, ec duplicaiton is designed to make duplicated copy at"},{"line_number":17,"context_line":"chunk_transformer function for each node. However, it can cause doubled"},{"line_number":18,"context_line":"etag calculation which should be expensive for Swift proxy work."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"ba2be162_7cf8814a","line":16,"range":{"start_line":16,"start_character":15,"end_line":16,"end_character":26},"updated":"2017-03-06 18:50:28.000000000","message":"nit: duplication","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":13,"context_line":"to know what\u0027s changed from the parent to reviewers so that if it could"},{"line_number":14,"context_line":"be okay to get this into the parent, I will squash this up to there."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Originally, ec duplicaiton is designed to make duplicated copy at"},{"line_number":17,"context_line":"chunk_transformer function for each node. However, it can cause doubled"},{"line_number":18,"context_line":"etag calculation which should be expensive for Swift proxy work."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"9a30ddce_2e0daa90","line":16,"range":{"start_line":16,"start_character":15,"end_line":16,"end_character":26},"in_reply_to":"ba2be162_7cf8814a","updated":"2017-03-08 09:21:07.000000000","message":"Done","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"62d2a645694b9b1d3c55012d416f8a16150006c2","unresolved":false,"context_lines":[{"line_number":1804,"context_line":"        last_frags \u003d policy.pyeclib_driver.encode(last_bytes)"},{"line_number":1805,"context_line":"        yield last_frags"},{"line_number":1806,"context_line":"    else:"},{"line_number":1807,"context_line":"        yield [\u0027\u0027] * nstreams"},{"line_number":1808,"context_line":""},{"line_number":1809,"context_line":""},{"line_number":1810,"context_line":"def trailing_metadata(policy, client_obj_hasher,"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_498ee1d0","line":1807,"updated":"2017-03-03 21:23:59.000000000","message":"nit: I\u0027m partial toward using policy.ec_n_unique_fragments here and scrapping nstreams altogether.","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"370f9b9f33624545e20aae39034b75d41c089025","unresolved":false,"context_lines":[{"line_number":1804,"context_line":"        last_frags \u003d policy.pyeclib_driver.encode(last_bytes)"},{"line_number":1805,"context_line":"        yield last_frags"},{"line_number":1806,"context_line":"    else:"},{"line_number":1807,"context_line":"        yield [\u0027\u0027] * nstreams"},{"line_number":1808,"context_line":""},{"line_number":1809,"context_line":""},{"line_number":1810,"context_line":"def trailing_metadata(policy, client_obj_hasher,"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_507bc07a","line":1807,"in_reply_to":"ba2be162_498ee1d0","updated":"2017-03-06 04:04:25.000000000","message":"Done","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":1747,"context_line":"                   mime_boundary, multiphase\u003dneed_multiphase)"},{"line_number":1748,"context_line":""},{"line_number":1749,"context_line":""},{"line_number":1750,"context_line":"def chunk_transformer(policy, nstreams):"},{"line_number":1751,"context_line":"    segment_size \u003d policy.ec_segment_size"},{"line_number":1752,"context_line":""},{"line_number":1753,"context_line":"    buf \u003d collections.deque()"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_e4422167","side":"PARENT","line":1750,"range":{"start_line":1750,"start_character":30,"end_line":1750,"end_character":38},"updated":"2017-03-06 18:50:28.000000000","message":"+1 makes sense to drop this arg","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":1802,"context_line":"    last_bytes \u003d \u0027\u0027.join(buf)"},{"line_number":1803,"context_line":"    if last_bytes:"},{"line_number":1804,"context_line":"        last_frags \u003d policy.pyeclib_driver.encode(last_bytes)"},{"line_number":1805,"context_line":"        yield last_frags * policy.ec_duplication_factor"},{"line_number":1806,"context_line":"    else:"},{"line_number":1807,"context_line":"        yield [\u0027\u0027] * nstreams"},{"line_number":1808,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_f64625be","side":"PARENT","line":1805,"updated":"2017-03-07 18:03:30.000000000","message":"whoa, this is big change.  Where is this now?  Oh, this whole function is now strongly oriented to frags instead of backend chunks","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":1802,"context_line":"    last_bytes \u003d \u0027\u0027.join(buf)"},{"line_number":1803,"context_line":"    if last_bytes:"},{"line_number":1804,"context_line":"        last_frags \u003d policy.pyeclib_driver.encode(last_bytes)"},{"line_number":1805,"context_line":"        yield last_frags * policy.ec_duplication_factor"},{"line_number":1806,"context_line":"    else:"},{"line_number":1807,"context_line":"        yield [\u0027\u0027] * nstreams"},{"line_number":1808,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_91c0777e","side":"PARENT","line":1805,"in_reply_to":"9a30ddce_f64625be","updated":"2017-03-08 09:21:07.000000000","message":"With this change, chunk_transformer is no longer needed to yield the duplicate fragments. Instead, send_chunk at L2406 can pick up the suitable fragment for each putters.","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2348,"context_line":"        # chunk 0, primary 1 gets chunk 1, and so on). For handoffs,"},{"line_number":2349,"context_line":"        # we assign the chunk index of a missing primary."},{"line_number":2350,"context_line":"        handoff_conns \u003d []"},{"line_number":2351,"context_line":"        chunk_index \u003d {}"},{"line_number":2352,"context_line":"        for p in putters:"},{"line_number":2353,"context_line":"            if p.node_index is not None:"},{"line_number":2354,"context_line":"                chunk_index[p] \u003d p.node_index"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_11ee638b","side":"PARENT","line":2351,"updated":"2017-03-07 18:03:30.000000000","message":"frag_index \u003d {}\n\nfrag_index[putter] \u003d policy.get_backend_index(p.node_index)\n\n^ requires no comment","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2421,"context_line":"                return"},{"line_number":2422,"context_line":""},{"line_number":2423,"context_line":"            for putter in list(putters):"},{"line_number":2424,"context_line":"                ci \u003d chunk_index[putter]"},{"line_number":2425,"context_line":"                backend_chunk \u003d backend_chunks[ci]"},{"line_number":2426,"context_line":"                if not putter.failed:"},{"line_number":2427,"context_line":"                    chunk_hashers[ci].update(backend_chunk)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_51445b73","side":"PARENT","line":2424,"updated":"2017-03-07 18:03:30.000000000","message":"fi \u003d frag_index[putter]","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2421,"context_line":"                return"},{"line_number":2422,"context_line":""},{"line_number":2423,"context_line":"            for putter in list(putters):"},{"line_number":2424,"context_line":"                ci \u003d chunk_index[putter]"},{"line_number":2425,"context_line":"                backend_chunk \u003d backend_chunks[ci]"},{"line_number":2426,"context_line":"                if not putter.failed:"},{"line_number":2427,"context_line":"                    chunk_hashers[ci].update(backend_chunk)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_0c90ee42","side":"PARENT","line":2424,"in_reply_to":"9a30ddce_51445b73","updated":"2017-03-08 09:21:07.000000000","message":"Done\n\nIn the new patch set the ci will be frag_index.","commit_id":"cf1c44dff0706ec4fa491aa7d3d956d3795ae675"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":1747,"context_line":"                   mime_boundary, multiphase\u003dneed_multiphase)"},{"line_number":1748,"context_line":""},{"line_number":1749,"context_line":""},{"line_number":1750,"context_line":"def chunk_transformer(policy):"},{"line_number":1751,"context_line":"    segment_size \u003d policy.ec_segment_size"},{"line_number":1752,"context_line":""},{"line_number":1753,"context_line":"    buf \u003d collections.deque()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_3117e7a4","line":1750,"updated":"2017-03-07 18:03:30.000000000","message":"I think a pre-existing docstring would have made it more immediately obvious this patch is a fundamental change in behavior for this function.\n\nNo longer do we have a generator that transform sent chunks into backend chunks - but instead very specifically it turns sent chunks into per fragment archive chunks - and only through careful handling in the caller can you make a set of backend chunks appropriate for a duplicated ec policy.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":1747,"context_line":"                   mime_boundary, multiphase\u003dneed_multiphase)"},{"line_number":1748,"context_line":""},{"line_number":1749,"context_line":""},{"line_number":1750,"context_line":"def chunk_transformer(policy):"},{"line_number":1751,"context_line":"    segment_size \u003d policy.ec_segment_size"},{"line_number":1752,"context_line":""},{"line_number":1753,"context_line":"    buf \u003d collections.deque()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_ae07da1a","line":1750,"in_reply_to":"9a30ddce_3117e7a4","updated":"2017-03-08 09:21:07.000000000","message":"I think your understanding is correct. Originally, chunk_transformer was designed to generate backend chunks which can be transferred to the object-server directly. But now, chunk_transformer is doing to build fragment archive chunks.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":1776,"context_line":"            frags_by_byte_order \u003d []"},{"line_number":1777,"context_line":"            for chunk_to_encode in chunks_to_encode:"},{"line_number":1778,"context_line":"                send_chunks \u003d policy.pyeclib_driver.encode(chunk_to_encode)"},{"line_number":1779,"context_line":"                frags_by_byte_order.append(send_chunks)"},{"line_number":1780,"context_line":"            # Sequential calls to encode() have given us a list that"},{"line_number":1781,"context_line":"            # looks like this:"},{"line_number":1782,"context_line":"            #"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_6492b1f7","line":1779,"updated":"2017-03-06 18:50:28.000000000","message":"nit: this could revert back to what we had before the EC frag duplication patch:\n\n                  frags_by_byte_order.append(\n                    policy.pyeclib_driver.encode(chunk_to_encode))","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":1776,"context_line":"            frags_by_byte_order \u003d []"},{"line_number":1777,"context_line":"            for chunk_to_encode in chunks_to_encode:"},{"line_number":1778,"context_line":"                send_chunks \u003d policy.pyeclib_driver.encode(chunk_to_encode)"},{"line_number":1779,"context_line":"                frags_by_byte_order.append(send_chunks)"},{"line_number":1780,"context_line":"            # Sequential calls to encode() have given us a list that"},{"line_number":1781,"context_line":"            # looks like this:"},{"line_number":1782,"context_line":"            #"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_d134cbdf","line":1779,"in_reply_to":"ba2be162_6492b1f7","updated":"2017-03-07 18:03:30.000000000","message":"Yeah I think w/o the `* policy.ec_duplication_factor` calling it \"send_chunks\" is confusing.\n\nIn fact aside from dropping nstreams this whole method is back to what it did pre-duplication factor?","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":2329,"context_line":"    def _determine_chunk_destinations(self, putters, policy):"},{"line_number":2330,"context_line":"        \"\"\""},{"line_number":2331,"context_line":"        Given a list of putters, return a dict where the key is the putter"},{"line_number":2332,"context_line":"        and the value is the node index to use."},{"line_number":2333,"context_line":""},{"line_number":2334,"context_line":"        This is done so that we line up handoffs using the same node index"},{"line_number":2335,"context_line":"        (in the primary part list) as the primary that the handoff is standing"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_24751963","line":2332,"range":{"start_line":2332,"start_character":29,"end_line":2332,"end_character":39},"updated":"2017-03-06 18:50:28.000000000","message":"this should be frag index now","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2329,"context_line":"    def _determine_chunk_destinations(self, putters, policy):"},{"line_number":2330,"context_line":"        \"\"\""},{"line_number":2331,"context_line":"        Given a list of putters, return a dict where the key is the putter"},{"line_number":2332,"context_line":"        and the value is the node index to use."},{"line_number":2333,"context_line":""},{"line_number":2334,"context_line":"        This is done so that we line up handoffs using the same node index"},{"line_number":2335,"context_line":"        (in the primary part list) as the primary that the handoff is standing"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_31ac2ba0","line":2332,"in_reply_to":"9a30ddce_f1f9cf4e","updated":"2017-03-08 09:21:07.000000000","message":"Done","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2329,"context_line":"    def _determine_chunk_destinations(self, putters, policy):"},{"line_number":2330,"context_line":"        \"\"\""},{"line_number":2331,"context_line":"        Given a list of putters, return a dict where the key is the putter"},{"line_number":2332,"context_line":"        and the value is the node index to use."},{"line_number":2333,"context_line":""},{"line_number":2334,"context_line":"        This is done so that we line up handoffs using the same node index"},{"line_number":2335,"context_line":"        (in the primary part list) as the primary that the handoff is standing"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_f1f9cf4e","line":2332,"in_reply_to":"ba2be162_24751963","updated":"2017-03-07 18:03:30.000000000","message":"+2!","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":2331,"context_line":"        Given a list of putters, return a dict where the key is the putter"},{"line_number":2332,"context_line":"        and the value is the node index to use."},{"line_number":2333,"context_line":""},{"line_number":2334,"context_line":"        This is done so that we line up handoffs using the same node index"},{"line_number":2335,"context_line":"        (in the primary part list) as the primary that the handoff is standing"},{"line_number":2336,"context_line":"        in for.  This lets erasure-code fragment archives wind up on the"},{"line_number":2337,"context_line":"        preferred local primary nodes when possible."}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_646b9185","line":2334,"range":{"start_line":2334,"start_character":64,"end_line":2334,"end_character":74},"updated":"2017-03-06 18:50:28.000000000","message":"frag index","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2331,"context_line":"        Given a list of putters, return a dict where the key is the putter"},{"line_number":2332,"context_line":"        and the value is the node index to use."},{"line_number":2333,"context_line":""},{"line_number":2334,"context_line":"        This is done so that we line up handoffs using the same node index"},{"line_number":2335,"context_line":"        (in the primary part list) as the primary that the handoff is standing"},{"line_number":2336,"context_line":"        in for.  This lets erasure-code fragment archives wind up on the"},{"line_number":2337,"context_line":"        preferred local primary nodes when possible."}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_11a567b4","line":2334,"range":{"start_line":2334,"start_character":64,"end_line":2334,"end_character":74},"in_reply_to":"ba2be162_646b9185","updated":"2017-03-08 09:21:07.000000000","message":"Done","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":2350,"context_line":"        chunk_index \u003d {}"},{"line_number":2351,"context_line":"        for p in putters:"},{"line_number":2352,"context_line":"            if p.node_index is not None:"},{"line_number":2353,"context_line":"                # Now we\u0027re using backend index as chunk index instead of"},{"line_number":2354,"context_line":"                # node_index"},{"line_number":2355,"context_line":"                chunk_index[p] \u003d policy.get_backend_index(p.node_index)"},{"line_number":2356,"context_line":"            else:"},{"line_number":2357,"context_line":"                handoff_conns.append(p)"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_a4bca9dc","line":2354,"range":{"start_line":2353,"start_character":18,"end_line":2354,"end_character":28},"updated":"2017-03-06 18:50:28.000000000","message":"I find this comment confusing, first because I still find \"backend\" index a confusing name for what is in effect frag index, but mainly because the comment only makes sense if you know the history of this method (i.e. that the chunk_index previously mapped putter-\u003enode_index but now maps putter-\u003efrag index","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2350,"context_line":"        chunk_index \u003d {}"},{"line_number":2351,"context_line":"        for p in putters:"},{"line_number":2352,"context_line":"            if p.node_index is not None:"},{"line_number":2353,"context_line":"                # Now we\u0027re using backend index as chunk index instead of"},{"line_number":2354,"context_line":"                # node_index"},{"line_number":2355,"context_line":"                chunk_index[p] \u003d policy.get_backend_index(p.node_index)"},{"line_number":2356,"context_line":"            else:"},{"line_number":2357,"context_line":"                handoff_conns.append(p)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_2cd77224","line":2354,"range":{"start_line":2353,"start_character":18,"end_line":2354,"end_character":28},"in_reply_to":"ba2be162_a4bca9dc","updated":"2017-03-08 09:21:07.000000000","message":"Perhaps, renaming get_backend_index to like get_backend_frag_index is better? I\u0027m now renaming the \"chunk_index\" dict to \"putter_to_frag_index\" to be intuitive name.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2352,"context_line":"            if p.node_index is not None:"},{"line_number":2353,"context_line":"                # Now we\u0027re using backend index as chunk index instead of"},{"line_number":2354,"context_line":"                # node_index"},{"line_number":2355,"context_line":"                chunk_index[p] \u003d policy.get_backend_index(p.node_index)"},{"line_number":2356,"context_line":"            else:"},{"line_number":2357,"context_line":"                handoff_conns.append(p)"},{"line_number":2358,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_71723fe7","line":2355,"updated":"2017-03-07 18:03:30.000000000","message":"This is what makes duplication possible, assigning multiple putters the same backend frag index.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2352,"context_line":"            if p.node_index is not None:"},{"line_number":2353,"context_line":"                # Now we\u0027re using backend index as chunk index instead of"},{"line_number":2354,"context_line":"                # node_index"},{"line_number":2355,"context_line":"                chunk_index[p] \u003d policy.get_backend_index(p.node_index)"},{"line_number":2356,"context_line":"            else:"},{"line_number":2357,"context_line":"                handoff_conns.append(p)"},{"line_number":2358,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_6ce16afe","line":2355,"in_reply_to":"9a30ddce_71723fe7","updated":"2017-03-08 09:21:07.000000000","message":"Correct.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":2363,"context_line":"        # returns 507, in which case a handoff is used to replace it."},{"line_number":2364,"context_line":""},{"line_number":2365,"context_line":"        # lack_list is a dict of list to keep hole indexes"},{"line_number":2366,"context_line":"        # e.g. if we have 2 holes for index 0 with ec_duplication_factor\u003d2"},{"line_number":2367,"context_line":"        # lack_list is like {0: [0], 1: [0]}, and then, if 1 hole found"},{"line_number":2368,"context_line":"        # for index 1, lack_list will be {0: [0, 1], 1: [0]}."},{"line_number":2369,"context_line":"        # After that, holes will be filled from bigger key"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_e4e5c1db","line":2366,"range":{"start_line":2366,"start_character":38,"end_line":2366,"end_character":43},"updated":"2017-03-06 18:50:28.000000000","message":"frag index","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2363,"context_line":"        # returns 507, in which case a handoff is used to replace it."},{"line_number":2364,"context_line":""},{"line_number":2365,"context_line":"        # lack_list is a dict of list to keep hole indexes"},{"line_number":2366,"context_line":"        # e.g. if we have 2 holes for index 0 with ec_duplication_factor\u003d2"},{"line_number":2367,"context_line":"        # lack_list is like {0: [0], 1: [0]}, and then, if 1 hole found"},{"line_number":2368,"context_line":"        # for index 1, lack_list will be {0: [0, 1], 1: [0]}."},{"line_number":2369,"context_line":"        # After that, holes will be filled from bigger key"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_2c8cd234","line":2366,"range":{"start_line":2366,"start_character":38,"end_line":2366,"end_character":43},"in_reply_to":"ba2be162_e4e5c1db","updated":"2017-03-08 09:21:07.000000000","message":"Done","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2424,"context_line":"                ci \u003d chunk_index[putter]"},{"line_number":2425,"context_line":"                backend_chunk \u003d backend_chunks[ci]"},{"line_number":2426,"context_line":"                if not putter.failed:"},{"line_number":2427,"context_line":"                    # N.B. chunk index is frag_index, not node_index"},{"line_number":2428,"context_line":"                    # so that same ci will appear when using"},{"line_number":2429,"context_line":"                    # ec_duplication_factor \u003e\u003d 2. So skip to feed the chunk"},{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_110583c1","line":2427,"updated":"2017-03-07 18:03:30.000000000","message":"so... rename the variable?","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2424,"context_line":"                ci \u003d chunk_index[putter]"},{"line_number":2425,"context_line":"                backend_chunk \u003d backend_chunks[ci]"},{"line_number":2426,"context_line":"                if not putter.failed:"},{"line_number":2427,"context_line":"                    # N.B. chunk index is frag_index, not node_index"},{"line_number":2428,"context_line":"                    # so that same ci will appear when using"},{"line_number":2429,"context_line":"                    # ec_duplication_factor \u003e\u003d 2. So skip to feed the chunk"},{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_b14f3bbc","line":2427,"in_reply_to":"9a30ddce_110583c1","updated":"2017-03-08 09:21:07.000000000","message":"Yes, definitely","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."},{"line_number":2431,"context_line":"                    if ci not in updated_chunk_indexes:"},{"line_number":2432,"context_line":"                        chunk_hashers[ci].update(backend_chunk)"},{"line_number":2433,"context_line":"                        updated_chunk_indexes.add(ci)"},{"line_number":2434,"context_line":"                    putter.send_chunk(backend_chunk)"},{"line_number":2435,"context_line":"                else:"},{"line_number":2436,"context_line":"                    putter.close()"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_443dad29","line":2433,"updated":"2017-03-06 18:50:28.000000000","message":"+1","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."},{"line_number":2431,"context_line":"                    if ci not in updated_chunk_indexes:"},{"line_number":2432,"context_line":"                        chunk_hashers[ci].update(backend_chunk)"},{"line_number":2433,"context_line":"                        updated_chunk_indexes.add(ci)"},{"line_number":2434,"context_line":"                    putter.send_chunk(backend_chunk)"},{"line_number":2435,"context_line":"                else:"},{"line_number":2436,"context_line":"                    putter.close()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_c74a31c5","line":2433,"in_reply_to":"9a30ddce_7505cac0","updated":"2017-03-08 09:21:07.000000000","message":"For now, obviously the node_index in the *proxy-server* is only used for referring the fragment index, IMHO. Otherwise, any other places can be broken with this change. To be more intuitive, it could be better to keep only frag_index in the Putter instance rather than node_index... it needs more line changes around _connect_put_node to take a policy as args, though.\n\n\nOn the other hand, the reason why we still have to keep \"node_index\" in the ring is because it is used in the reconstructor to determine which *node* should do the reconstruction work. However it\u0027s isolated from the proxy work here. What proxy should know is that which frag_index each putter should transfer to the assigned object-server, right?","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8881a468af3e42b95b5853eae328079fc297a335","unresolved":false,"context_lines":[{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."},{"line_number":2431,"context_line":"                    if ci not in updated_chunk_indexes:"},{"line_number":2432,"context_line":"                        chunk_hashers[ci].update(backend_chunk)"},{"line_number":2433,"context_line":"                        updated_chunk_indexes.add(ci)"},{"line_number":2434,"context_line":"                    putter.send_chunk(backend_chunk)"},{"line_number":2435,"context_line":"                else:"},{"line_number":2436,"context_line":"                    putter.close()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_39732082","line":2433,"in_reply_to":"9a30ddce_c74a31c5","updated":"2017-03-08 15:59:27.000000000","message":"I can see it both ways, but for me it is more intuitive to think of chunk_transformer transforming from an input chunk to however many fragment archives the pyeclib driver produces. Duplication is not a property of the pyeclib driver, so for me it makes sense that duplication is implemented by having Nx putters and those being \"configured\" via putters_to_frag_index to select the correct driver frag archive for their respective nodes.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2430,"context_line":"                    # to hasher if the ci was updated already."},{"line_number":2431,"context_line":"                    if ci not in updated_chunk_indexes:"},{"line_number":2432,"context_line":"                        chunk_hashers[ci].update(backend_chunk)"},{"line_number":2433,"context_line":"                        updated_chunk_indexes.add(ci)"},{"line_number":2434,"context_line":"                    putter.send_chunk(backend_chunk)"},{"line_number":2435,"context_line":"                else:"},{"line_number":2436,"context_line":"                    putter.close()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_7505cac0","line":2433,"in_reply_to":"ba2be162_443dad29","updated":"2017-03-07 18:03:30.000000000","message":"FWIW I think we could have done this same skip thing without changing the nature of the chunk_transformer\n\ninstead of chunk_hashers\u0027 we\u0027d have frag_hashers\n\n                if not putter.failed:\n                    # only update hashes per frag\n                    frag_index \u003d policy.get_backend_index(ci)\n                    if frag_index not in hashed_frag_indexes:\n                        frag_hashers[frag_index].update(backend_chunk)\n                        hashed_frag_indexes.add(frag_index)\n                    putter.send_chunk(backend_chunk)\n\n\nUnless we\u0027re sure moving the backend chunk index business from node_index to frag_index is more obvious/intuitive/maintainable in the long run than the chunk_transformer doing the duplication?","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e87f9e4ae84afd755559f571af2ed1621716219d","unresolved":false,"context_lines":[{"line_number":2497,"context_line":"                    # Update any footers set by middleware with EC footers"},{"line_number":2498,"context_line":"                    trail_md \u003d trailing_metadata("},{"line_number":2499,"context_line":"                        policy, etag_hasher,"},{"line_number":2500,"context_line":"                        bytes_transferred, ci)"},{"line_number":2501,"context_line":"                    trail_md.update(footers)"},{"line_number":2502,"context_line":"                    # Etag footer must always be hash of what we sent"},{"line_number":2503,"context_line":"                    trail_md[\u0027Etag\u0027] \u003d chunk_hashers[ci].hexdigest()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_9195f341","line":2500,"updated":"2017-03-07 18:03:30.000000000","message":"yeah, I\u0027m not sure how we can maintain this\n\nwe need to audit everywhere we talk about a \"chunk index\" - and either call it frag index or a node index.\n\nAlso, isn\u0027t get_backend_index the identify function for frag_index?\n\nfrag_index \u003d\u003d policy.get_backend_index(frag_index)","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":2497,"context_line":"                    # Update any footers set by middleware with EC footers"},{"line_number":2498,"context_line":"                    trail_md \u003d trailing_metadata("},{"line_number":2499,"context_line":"                        policy, etag_hasher,"},{"line_number":2500,"context_line":"                        bytes_transferred, ci)"},{"line_number":2501,"context_line":"                    trail_md.update(footers)"},{"line_number":2502,"context_line":"                    # Etag footer must always be hash of what we sent"},{"line_number":2503,"context_line":"                    trail_md[\u0027Etag\u0027] \u003d chunk_hashers[ci].hexdigest()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_e74ad5ab","line":2500,"in_reply_to":"9a30ddce_9195f341","updated":"2017-03-08 09:21:07.000000000","message":"The ci will be replaced with frag_index in the new patch set. Does it make you sense?","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"884380dcb25cd51c00ae9279f6a2b67900bf663b","unresolved":false,"context_lines":[{"line_number":1764,"context_line":"        total_buf_len +\u003d len(chunk)"},{"line_number":1765,"context_line":"        if total_buf_len \u003e\u003d segment_size:"},{"line_number":1766,"context_line":"            chunks_to_encode \u003d []"},{"line_number":1767,"context_line":"            # extract as many chunks as we can from the input buffer"},{"line_number":1768,"context_line":"            while total_buf_len \u003e\u003d segment_size:"},{"line_number":1769,"context_line":"                to_take \u003d segment_size"},{"line_number":1770,"context_line":"                pieces \u003d []"}],"source_content_type":"text/x-python","patch_set":10,"id":"9a30ddce_3e6946bf","line":1767,"updated":"2017-03-08 16:43:04.000000000","message":"nit: I *think* we are actually extracting segments from the input buffer, at least we\u0027re extracting an integer multiple of segment_size\n\nin this method we have an input \u0027chunk\u0027 which is buffered, then aligned on segment_size into \u0027chunks_to_encode\u0027, which are fed to the EC engine to get \u0027frags_by_byte_order\u0027, which are each joined to form \u0027obj_data\u0027, which the caller assigns to a variable \u0027backend_chunks\u0027 !","commit_id":"4187c839c3ddd4b256b4ece077ee293322f96af9"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"62d2a645694b9b1d3c55012d416f8a16150006c2","unresolved":false,"context_lines":[{"line_number":1454,"context_line":"        got \u003d controller._determine_chunk_destinations(putters, self.policy)"},{"line_number":1455,"context_line":"        expected \u003d {}"},{"line_number":1456,"context_line":"        for i, p in enumerate(putters):"},{"line_number":1457,"context_line":"            # expected[p] \u003d i"},{"line_number":1458,"context_line":"            expected[p] \u003d self.policy.get_backend_index(i)"},{"line_number":1459,"context_line":"        self.assertEqual(got, expected)"},{"line_number":1460,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_d47ee468","line":1457,"updated":"2017-03-03 21:23:59.000000000","message":"We should just get rid of this now; leaving it commented out will only confuse us later.","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"370f9b9f33624545e20aae39034b75d41c089025","unresolved":false,"context_lines":[{"line_number":1454,"context_line":"        got \u003d controller._determine_chunk_destinations(putters, self.policy)"},{"line_number":1455,"context_line":"        expected \u003d {}"},{"line_number":1456,"context_line":"        for i, p in enumerate(putters):"},{"line_number":1457,"context_line":"            # expected[p] \u003d i"},{"line_number":1458,"context_line":"            expected[p] \u003d self.policy.get_backend_index(i)"},{"line_number":1459,"context_line":"        self.assertEqual(got, expected)"},{"line_number":1460,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_9099b8c6","line":1457,"in_reply_to":"ba2be162_d47ee468","updated":"2017-03-06 04:04:25.000000000","message":"Oops, my fault, thanks for catching up! Done.","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"62d2a645694b9b1d3c55012d416f8a16150006c2","unresolved":false,"context_lines":[{"line_number":4335,"context_line":"        # index one_more_missing should not be choosen so only 1"},{"line_number":4336,"context_line":"        # index (another one as duplicated) should be found"},{"line_number":4337,"context_line":"        self.assertEqual(1, got.values().count("},{"line_number":4338,"context_line":"            self.policy.get_backend_index(one_more_missing)))"},{"line_number":4339,"context_line":"        # either index unique or duplicated should be in the got dict as"},{"line_number":4340,"context_line":"        # a unique_index"},{"line_number":4341,"context_line":"        self.assertTrue(1, got.values().count(unique))"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_4f1501e3","line":4338,"updated":"2017-03-03 21:23:59.000000000","message":"This almost seems like a change in behavior... I suppose before this was showing up as one_more_missing + ec_n_unique_fragments? And now we just want to make sure we have 1 and not 2.\n\nI was a little confused about what we\u0027re really trying to test here. I wonder if something like http://paste.openstack.org/show/601374/ might be a bit more clear?","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"370f9b9f33624545e20aae39034b75d41c089025","unresolved":false,"context_lines":[{"line_number":4335,"context_line":"        # index one_more_missing should not be choosen so only 1"},{"line_number":4336,"context_line":"        # index (another one as duplicated) should be found"},{"line_number":4337,"context_line":"        self.assertEqual(1, got.values().count("},{"line_number":4338,"context_line":"            self.policy.get_backend_index(one_more_missing)))"},{"line_number":4339,"context_line":"        # either index unique or duplicated should be in the got dict as"},{"line_number":4340,"context_line":"        # a unique_index"},{"line_number":4341,"context_line":"        self.assertTrue(1, got.values().count(unique))"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba2be162_10a4e8b7","line":4338,"in_reply_to":"ba2be162_4f1501e3","updated":"2017-03-06 04:04:25.000000000","message":"Awesome! It\u0027s nice for us (at least me) to understand the state.","commit_id":"2233359066c4b5fd3458facaff089b3dbbe93c9c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":1454,"context_line":"        got \u003d controller._determine_chunk_destinations(putters, self.policy)"},{"line_number":1455,"context_line":"        expected \u003d {}"},{"line_number":1456,"context_line":"        for i, p in enumerate(putters):"},{"line_number":1457,"context_line":"            expected[p] \u003d self.policy.get_backend_index(i)"},{"line_number":1458,"context_line":"        self.assertEqual(got, expected)"},{"line_number":1459,"context_line":""},{"line_number":1460,"context_line":"        # now lets make a handoff at the end"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_3fc8c254","line":1457,"updated":"2017-03-06 18:50:28.000000000","message":"nit: this change does not get tested i.e. the test is not run with a duplication factor \u003e1","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":1454,"context_line":"        got \u003d controller._determine_chunk_destinations(putters, self.policy)"},{"line_number":1455,"context_line":"        expected \u003d {}"},{"line_number":1456,"context_line":"        for i, p in enumerate(putters):"},{"line_number":1457,"context_line":"            expected[p] \u003d self.policy.get_backend_index(i)"},{"line_number":1458,"context_line":"        self.assertEqual(got, expected)"},{"line_number":1459,"context_line":""},{"line_number":1460,"context_line":"        # now lets make a handoff at the end"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_47a00144","line":1457,"in_reply_to":"ba2be162_3fc8c254","updated":"2017-03-08 09:21:07.000000000","message":"Nice catch, done.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":3656,"context_line":"    def test_chunk_transformer(self):"},{"line_number":3657,"context_line":"        def do_test(dup):"},{"line_number":3658,"context_line":"            segment_size \u003d 1024"},{"line_number":3659,"context_line":"            orig_chunk \u003d \u0027a\u0027 * segment_size"},{"line_number":3660,"context_line":"            policy \u003d ECStoragePolicy(0, \u0027ec8-2\u0027, ec_type\u003dDEFAULT_TEST_EC_TYPE,"},{"line_number":3661,"context_line":"                                     ec_ndata\u003d8, ec_nparity\u003d2,"},{"line_number":3662,"context_line":"                                     object_ring\u003dFakeRing(replicas\u003d10 * dup),"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_e8d4b3be","line":3659,"updated":"2017-03-06 18:50:28.000000000","message":"off-topic - we don\u0027t seem to have any test for orig_chunk larger than one segment size.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8881a468af3e42b95b5853eae328079fc297a335","unresolved":false,"context_lines":[{"line_number":3656,"context_line":"    def test_chunk_transformer(self):"},{"line_number":3657,"context_line":"        def do_test(dup):"},{"line_number":3658,"context_line":"            segment_size \u003d 1024"},{"line_number":3659,"context_line":"            orig_chunk \u003d \u0027a\u0027 * segment_size"},{"line_number":3660,"context_line":"            policy \u003d ECStoragePolicy(0, \u0027ec8-2\u0027, ec_type\u003dDEFAULT_TEST_EC_TYPE,"},{"line_number":3661,"context_line":"                                     ec_ndata\u003d8, ec_nparity\u003d2,"},{"line_number":3662,"context_line":"                                     object_ring\u003dFakeRing(replicas\u003d10 * dup),"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_f98268ff","line":3659,"in_reply_to":"9a30ddce_625b332f","updated":"2017-03-08 15:59:27.000000000","message":"yes and that patch could be squashed into this , but perhaps now we should get this one landed without further complicating","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":3656,"context_line":"    def test_chunk_transformer(self):"},{"line_number":3657,"context_line":"        def do_test(dup):"},{"line_number":3658,"context_line":"            segment_size \u003d 1024"},{"line_number":3659,"context_line":"            orig_chunk \u003d \u0027a\u0027 * segment_size"},{"line_number":3660,"context_line":"            policy \u003d ECStoragePolicy(0, \u0027ec8-2\u0027, ec_type\u003dDEFAULT_TEST_EC_TYPE,"},{"line_number":3661,"context_line":"                                     ec_ndata\u003d8, ec_nparity\u003d2,"},{"line_number":3662,"context_line":"                                     object_ring\u003dFakeRing(replicas\u003d10 * dup),"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_625b332f","line":3659,"in_reply_to":"ba2be162_e8d4b3be","updated":"2017-03-08 09:21:07.000000000","message":"Ah, that\u0027s what you\u0027re working to resolve with https://review.openstack.org/#/c/442108/\n\nMake sense.","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e33b0b20bc5bd333062dacb5b1563b608ceb23d3","unresolved":false,"context_lines":[{"line_number":4319,"context_line":""},{"line_number":4320,"context_line":"        # and then, pop a fragment which has same fragment index with unique"},{"line_number":4321,"context_line":"        self.assertEqual("},{"line_number":4322,"context_line":"            unique, self.policy.get_backend_index(duplicated))  # sanity"},{"line_number":4323,"context_line":"        putters.pop(duplicated)"},{"line_number":4324,"context_line":""},{"line_number":4325,"context_line":"        # pop one more fragment too to make one missing hole"}],"source_content_type":"text/x-python","patch_set":9,"id":"ba2be162_5119e621","line":4322,"updated":"2017-03-06 18:50:28.000000000","message":"rather than passing in duplicated and then making this assertion, why not calculate duplicated from unique:\n\n  duplicated \u003d unique + policy.ec.num_unique_fragments\n\n(see my follow up patch)","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f77cbc340f012185b5101a8186ad73089cebb091","unresolved":false,"context_lines":[{"line_number":4319,"context_line":""},{"line_number":4320,"context_line":"        # and then, pop a fragment which has same fragment index with unique"},{"line_number":4321,"context_line":"        self.assertEqual("},{"line_number":4322,"context_line":"            unique, self.policy.get_backend_index(duplicated))  # sanity"},{"line_number":4323,"context_line":"        putters.pop(duplicated)"},{"line_number":4324,"context_line":""},{"line_number":4325,"context_line":"        # pop one more fragment too to make one missing hole"}],"source_content_type":"text/x-python","patch_set":9,"id":"9a30ddce_6245f315","line":4322,"in_reply_to":"ba2be162_5119e621","updated":"2017-03-08 09:21:07.000000000","message":"Good call and I confirmed that is in your https://review.openstack.org/#/c/442108/ .\n\nSounds good improvement to me. Thanks!","commit_id":"73bedd881db202d02531e817e1b3d00c1cc7e0a3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"884380dcb25cd51c00ae9279f6a2b67900bf663b","unresolved":false,"context_lines":[{"line_number":1454,"context_line":"        got \u003d controller._determine_chunk_destinations(putters, self.policy)"},{"line_number":1455,"context_line":"        expected \u003d {}"},{"line_number":1456,"context_line":"        for i, p in enumerate(putters):"},{"line_number":1457,"context_line":"            expected[p] \u003d i"},{"line_number":1458,"context_line":"        self.assertEqual(got, expected)"},{"line_number":1459,"context_line":""},{"line_number":1460,"context_line":"        # now lets make a handoff at the end"}],"source_content_type":"text/x-python","patch_set":10,"id":"9a30ddce_99f88c4b","line":1457,"updated":"2017-03-08 16:43:04.000000000","message":"ok, we have a check here [1] that the duplicated putters get mapped to the correct frag index\n\n[1] https://github.com/openstack/swift/blob/4187c839c3ddd4b256b4ece077ee293322f96af9/test/unit/proxy/controllers/test_obj.py#L4311-L4314","commit_id":"4187c839c3ddd4b256b4ece077ee293322f96af9"}]}