)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d0a32cc483ec29f8755c3bde1826ba5f364bb617","unresolved":false,"context_lines":[{"line_number":21,"context_line":" * perform metadata check when a read starts on any frag_size"},{"line_number":22,"context_line":"   boundary, not just at zero"},{"line_number":23,"context_line":" * better buffer handling in _check_frag to reduce memcpy from"},{"line_number":24,"context_line":"   string slicing"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Related-Bug: #1631144"},{"line_number":27,"context_line":"Closes-Bug: #1633647"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"ba5da102_03f8667a","line":24,"updated":"2016-11-01 20:10:55.000000000","message":"I think the buffer handling is *pretty* good now, maybe it could still be optimized, but I think it\u0027s decidedly not terrible - probably pull this out.","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5f698d5ff9d9fb21425383410c44207fc790c076","unresolved":false,"context_lines":[{"line_number":1621,"context_line":"            while True:"},{"line_number":1622,"context_line":"                chunk \u003d self._fp.read(self._disk_chunk_size)"},{"line_number":1623,"context_line":"                if chunk:"},{"line_number":1624,"context_line":"                    self._update_checks(chunk)"},{"line_number":1625,"context_line":"                    self._bytes_read +\u003d len(chunk)"},{"line_number":1626,"context_line":"                    if self._bytes_read - dropped_cache \u003e DROP_CACHE_WINDOW:"},{"line_number":1627,"context_line":"                        self._drop_cache(self._fp.fileno(), dropped_cache,"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_9267dbe9","line":1624,"updated":"2016-10-18 02:55:23.000000000","message":"I think this looks fine compared to trying to wrap __iter__ - I think the logic is easier to reason about closer to _fp.read land than in __iter__/generator land.","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"733553563f6c7a49840775244fb1f6fc70cd67e5","unresolved":false,"context_lines":[{"line_number":2584,"context_line":"        return hashes[None].hexdigest()"},{"line_number":2585,"context_line":""},{"line_number":2586,"context_line":""},{"line_number":2587,"context_line":"class ECDiskFileReader(BaseDiskFileReader):"},{"line_number":2588,"context_line":"    def __init__(self, fp, data_file, obj_size, etag,"},{"line_number":2589,"context_line":"                 disk_chunk_size, keep_cache_size, device_path, logger,"},{"line_number":2590,"context_line":"                 quarantine_hook, use_splice, pipe_size, diskfile,"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_2b0ea3e5","line":2587,"updated":"2016-10-17 19:59:05.000000000","message":"at first I tried to wrap the super class __iter__, but TBH I just don\u0027t know how to get that to work in python, so went for this approach.","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"733553563f6c7a49840775244fb1f6fc70cd67e5","unresolved":false,"context_lines":[{"line_number":2606,"context_line":"    def _check_frag(self, frag):"},{"line_number":2607,"context_line":"        try:"},{"line_number":2608,"context_line":"            self._diskfile.policy.pyeclib_driver.get_metadata(frag)"},{"line_number":2609,"context_line":"        except ECDriverError as err:"},{"line_number":2610,"context_line":"            self.frag_errors.append((self.frag_buf_pointer, err))"},{"line_number":2611,"context_line":""},{"line_number":2612,"context_line":"    def _update_checks(self, chunk):"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_4b2af72d","line":2609,"updated":"2016-10-17 19:59:05.000000000","message":"we could blow up and quarantine as soon as one bad frag is found but this sits better with the current quarantine logic so i took the easy way out. Plus, we\u0027re not expecting this to happen often.","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5f698d5ff9d9fb21425383410c44207fc790c076","unresolved":false,"context_lines":[{"line_number":2606,"context_line":"    def _check_frag(self, frag):"},{"line_number":2607,"context_line":"        try:"},{"line_number":2608,"context_line":"            self._diskfile.policy.pyeclib_driver.get_metadata(frag)"},{"line_number":2609,"context_line":"        except ECDriverError as err:"},{"line_number":2610,"context_line":"            self.frag_errors.append((self.frag_buf_pointer, err))"},{"line_number":2611,"context_line":""},{"line_number":2612,"context_line":"    def _update_checks(self, chunk):"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_5208e328","line":2609,"in_reply_to":"fa6399be_4b2af72d","updated":"2016-10-18 02:55:23.000000000","message":"agree with sam - would love to just quarantine here and then blow up, also like the idea of not having to keep the bad frag context in state","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5f698d5ff9d9fb21425383410c44207fc790c076","unresolved":false,"context_lines":[{"line_number":2616,"context_line":"            while len(self.frag_buf) \u003e\u003d self.frag_size:"},{"line_number":2617,"context_line":"                self._check_frag(self.frag_buf[:self.frag_size])"},{"line_number":2618,"context_line":"                self.frag_buf \u003d self.frag_buf[self.frag_size:]"},{"line_number":2619,"context_line":"                self.frag_buf_pointer +\u003d self.frag_size"},{"line_number":2620,"context_line":""},{"line_number":2621,"context_line":"    def _handle_close_quarantine(self):"},{"line_number":2622,"context_line":"        super(ECDiskFileReader, self)._handle_close_quarantine()"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_d24a3355","line":2619,"updated":"2016-10-18 02:55:23.000000000","message":"I think we could avoid some memcpy with just some array index math","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"733553563f6c7a49840775244fb1f6fc70cd67e5","unresolved":false,"context_lines":[{"line_number":2624,"context_line":"            self._check_frag(self.frag_buf)"},{"line_number":2625,"context_line":"        if self.frag_errors:"},{"line_number":2626,"context_line":"            self._quarantine(\u0027Bad EC fragment metadata found at offset %s\u0027 %"},{"line_number":2627,"context_line":"                             self.frag_errors[0][0])"},{"line_number":2628,"context_line":""},{"line_number":2629,"context_line":""},{"line_number":2630,"context_line":"class ECDiskFileWriter(BaseDiskFileWriter):"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_8b1c0f15","line":2627,"updated":"2016-10-17 19:59:05.000000000","message":"we could report them all, since we stashed them all, but w/e","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"75d169f4a1dc09d26eef275b321fa74ed1b66a4a","unresolved":false,"context_lines":[{"line_number":2601,"context_line":"    def _init_checks(self):"},{"line_number":2602,"context_line":"        super(ECDiskFileReader, self)._init_checks()"},{"line_number":2603,"context_line":"        if self._started_at_0:"},{"line_number":2604,"context_line":"            self.frag_buf \u003d \u0027\u0027"},{"line_number":2605,"context_line":""},{"line_number":2606,"context_line":"    def _check_frag(self, frag):"},{"line_number":2607,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_df7313e6","line":2604,"updated":"2016-10-18 09:11:07.000000000","message":"yeah it\u0027d be great if there was a way to detect any frag boundary - but that\u0027s a wishlist priority to me at best - not needed for the backport","commit_id":"4c316f55d4e8e4d34e7ce3242d571785a684feeb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"75d169f4a1dc09d26eef275b321fa74ed1b66a4a","unresolved":false,"context_lines":[{"line_number":2608,"context_line":"            self._diskfile.policy.pyeclib_driver.get_metadata(frag)"},{"line_number":2609,"context_line":"        except ECDriverError:"},{"line_number":2610,"context_line":"            self._quarantine("},{"line_number":2611,"context_line":"                \u0027Invalid EC metadata at offset 0x%x\u0027 % self.frag_buf_pointer)"},{"line_number":2612,"context_line":""},{"line_number":2613,"context_line":"    def _update_checks(self, chunk):"},{"line_number":2614,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_8ac957af","line":2611,"updated":"2016-10-18 09:11:07.000000000","message":"normally when a diskfile calls it\u0027s _quarantine (not to be confused with dfm._quarantine) it does *not* raise.\n\nThe auditor installs a quarantine_hook that *does* raise tho - and that works good.\n\nIn object-server land we\u0027re is already in eventlet.wsgi land - I think I\u0027m just renaming the object but otherwise still letting the bytes out.  Probably need to call halt the iterator and close file pointer somehow.","commit_id":"4c316f55d4e8e4d34e7ce3242d571785a684feeb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"75d169f4a1dc09d26eef275b321fa74ed1b66a4a","unresolved":false,"context_lines":[{"line_number":2617,"context_line":"            while len(self.frag_buf) \u003e\u003d self.frag_size:"},{"line_number":2618,"context_line":"                self._check_frag(self.frag_buf[:self.frag_size])"},{"line_number":2619,"context_line":"                self.frag_buf \u003d self.frag_buf[self.frag_size:]"},{"line_number":2620,"context_line":"                self.frag_buf_pointer +\u003d self.frag_size"},{"line_number":2621,"context_line":""},{"line_number":2622,"context_line":"    def _handle_close_quarantine(self):"},{"line_number":2623,"context_line":"        super(ECDiskFileReader, self)._handle_close_quarantine()"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_bf881fc8","line":2620,"updated":"2016-10-18 09:11:07.000000000","message":"I didn\u0027t play with buffer optimization - seems silly to spend time trying to eliminate a memcpy when there\u0027s tests to fix","commit_id":"4c316f55d4e8e4d34e7ce3242d571785a684feeb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d245a58306565da572c69279cb5b621641027464","unresolved":false,"context_lines":[{"line_number":2608,"context_line":"        if self._started_at_0:"},{"line_number":2609,"context_line":"            self.frag_buf \u003d \u0027\u0027"},{"line_number":2610,"context_line":"        else:"},{"line_number":2611,"context_line":"            self.frag_buf \u003d None"},{"line_number":2612,"context_line":""},{"line_number":2613,"context_line":"    def _check_frag(self, frag):"},{"line_number":2614,"context_line":"        if not frag:"}],"source_content_type":"text/x-python","patch_set":6,"id":"fa6399be_389d9066","line":2611,"updated":"2016-10-21 10:52:54.000000000","message":"this fixed failing tests in proxy/test_server","commit_id":"b7d61407c6bb90a90c50821db2dbd085064befd4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d245a58306565da572c69279cb5b621641027464","unresolved":false,"context_lines":[{"line_number":2613,"context_line":"    def _check_frag(self, frag):"},{"line_number":2614,"context_line":"        if not frag:"},{"line_number":2615,"context_line":"            return"},{"line_number":2616,"context_line":"        if not isinstance(frag, six.binary_type):"},{"line_number":2617,"context_line":"            # ECInvalidParameter can be returned if the frag violates the input"},{"line_number":2618,"context_line":"            # format so for safety, check the input chunk if it\u0027s binary to"},{"line_number":2619,"context_line":"            # avoid quarantining a valid fragment archive."}],"source_content_type":"text/x-python","patch_set":6,"id":"fa6399be_d884940c","line":2616,"updated":"2016-10-21 10:52:54.000000000","message":"no test coverage for this yet","commit_id":"b7d61407c6bb90a90c50821db2dbd085064befd4"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":2608,"context_line":"        if self._started_at_0:"},{"line_number":2609,"context_line":"            self.frag_buf \u003d \u0027\u0027"},{"line_number":2610,"context_line":"        else:"},{"line_number":2611,"context_line":"            self.frag_buf \u003d None"},{"line_number":2612,"context_line":""},{"line_number":2613,"context_line":"    def _check_frag(self, frag):"},{"line_number":2614,"context_line":"        if not frag:"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_c78efd26","line":2611,"updated":"2016-10-21 08:50:10.000000000","message":"Ok, this is a fix for test/unit/proxy/test_server.py. frag_buf should be None to skip _check_frag in ECDsikFileReader. And TODO comment is also worth. If the range iter call starts to the fragment boundary (swift should do for EC case), we can check the fragment integrity from the boundary.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":2608,"context_line":"        if self._started_at_0:"},{"line_number":2609,"context_line":"            self.frag_buf \u003d \u0027\u0027"},{"line_number":2610,"context_line":"        else:"},{"line_number":2611,"context_line":"            self.frag_buf \u003d None"},{"line_number":2612,"context_line":""},{"line_number":2613,"context_line":"    def _check_frag(self, frag):"},{"line_number":2614,"context_line":"        if not frag:"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_a6ff4133","line":2611,"in_reply_to":"da6895a0_c78efd26","updated":"2016-10-21 12:15:46.000000000","message":"Yes, this fixed the proxy/test_server test failures, and I added a unit test in test_diskfile.py to verify this too","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a1ac995832c6b60dfb7b2a6772216e30f507d929","unresolved":false,"context_lines":[{"line_number":2622,"context_line":"                  \u0027%(datadir)s: %(type)s at offset 0x%(offset)x\u0027),"},{"line_number":2623,"context_line":"                {\u0027datadir\u0027: self._diskfile._datadir,"},{"line_number":2624,"context_line":"                 \u0027type\u0027: type(frag),"},{"line_number":2625,"context_line":"                 \u0027offset\u0027: self.frag_buf_pointer})"},{"line_number":2626,"context_line":""},{"line_number":2627,"context_line":"        try:"},{"line_number":2628,"context_line":"            self._diskfile.policy.pyeclib_driver.get_metadata(frag)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_b65221dc","line":2625,"updated":"2016-10-21 01:35:26.000000000","message":"Do\n\n return\n\nhere. Otherwise, we can feed the not-binary_type frag to get_metadata and that will trigger quarantine.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e85a0d0d0434bba9cbfd25b173b8bc092c90700a","unresolved":false,"context_lines":[{"line_number":2622,"context_line":"                  \u0027%(datadir)s: %(type)s at offset 0x%(offset)x\u0027),"},{"line_number":2623,"context_line":"                {\u0027datadir\u0027: self._diskfile._datadir,"},{"line_number":2624,"context_line":"                 \u0027type\u0027: type(frag),"},{"line_number":2625,"context_line":"                 \u0027offset\u0027: self.frag_buf_pointer})"},{"line_number":2626,"context_line":""},{"line_number":2627,"context_line":"        try:"},{"line_number":2628,"context_line":"            self._diskfile.policy.pyeclib_driver.get_metadata(frag)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_16fe6d6b","line":2625,"updated":"2016-10-21 01:48:08.000000000","message":"So I\u0027ve only just started looking at this, but I dont quite understand this bit. You check to see if the frag is a binary type or not saying that ECInvalidParam can be raised if violates the input format.. so you log a warning saying it\u0027s not quarantined.. but then couldn\u0027t then just go get quarantined down in the next block? I guess I just don\u0027t understand why check? Where is the safety here? Log if there could be a problem.. but then what are you going to do about it?\n\nI guess what I mean is how does this avoid quarantining a valid fragment chunk?\n\nIs something suppose to break or return here?","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"42e6f07b164645c745db646937b89eda6d074db2","unresolved":false,"context_lines":[{"line_number":2649,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"},{"line_number":2650,"context_line":"        if self.frag_buf is not None:"},{"line_number":2651,"context_line":"            self.frag_buf +\u003d chunk"},{"line_number":2652,"context_line":"            cursor \u003d 0"},{"line_number":2653,"context_line":"            while len(self.frag_buf) \u003e\u003d cursor + self.frag_size:"},{"line_number":2654,"context_line":"                self._check_frag(self.frag_buf[cursor:cursor + self.frag_size])"},{"line_number":2655,"context_line":"                cursor +\u003d self.frag_size"},{"line_number":2656,"context_line":"                self.frag_offset +\u003d self.frag_size"},{"line_number":2657,"context_line":"            if cursor:"},{"line_number":2658,"context_line":"                self.frag_buf \u003d self.frag_buf[cursor:]"},{"line_number":2659,"context_line":""},{"line_number":2660,"context_line":"    def _handle_close_quarantine(self):"},{"line_number":2661,"context_line":"        super(ECDiskFileReader, self)._handle_close_quarantine()"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_7844e173","line":2658,"range":{"start_line":2652,"start_character":0,"end_line":2658,"end_character":54},"updated":"2016-10-21 13:49:42.000000000","message":"There\u0027s still a copy that could be avoided with a circular buffer or double buffering.\n\n@Clay @Kota if you don\u0027t like this change then feel free to back it out. I just thought that it\u0027s worth making it a little better upfront since its going to be backported","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e5ca6b14b309064b68c0ae47b9aa899948065500","unresolved":false,"context_lines":[{"line_number":2655,"context_line":"                cursor +\u003d self.frag_size"},{"line_number":2656,"context_line":"                self.frag_offset +\u003d self.frag_size"},{"line_number":2657,"context_line":"            if cursor:"},{"line_number":2658,"context_line":"                self.frag_buf \u003d self.frag_buf[cursor:]"},{"line_number":2659,"context_line":""},{"line_number":2660,"context_line":"    def _handle_close_quarantine(self):"},{"line_number":2661,"context_line":"        super(ECDiskFileReader, self)._handle_close_quarantine()"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_8df888e7","line":2658,"in_reply_to":"da6895a0_7844e173","updated":"2016-10-21 19:52:54.000000000","message":"Yeah, no this is great!","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d0a32cc483ec29f8755c3bde1826ba5f364bb617","unresolved":false,"context_lines":[{"line_number":2604,"context_line":"        self.frag_buf \u003d None"},{"line_number":2605,"context_line":"        self.frag_offset \u003d 0"},{"line_number":2606,"context_line":"        self.frag_size \u003d self._diskfile.policy.fragment_size"},{"line_number":2607,"context_line":"        self.frag_errors \u003d []"},{"line_number":2608,"context_line":""},{"line_number":2609,"context_line":"    def _init_checks(self):"},{"line_number":2610,"context_line":"        super(ECDiskFileReader, self)._init_checks()"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_2e19d179","line":2607,"updated":"2016-11-01 20:10:55.000000000","message":"nit: not used aymore","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1d0b595b5a9dd824ce9ce1150c0dfa53f097c89f","unresolved":false,"context_lines":[{"line_number":2655,"context_line":"    def _update_checks(self, chunk):"},{"line_number":2656,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"},{"line_number":2657,"context_line":"        if self.frag_buf is not None:"},{"line_number":2658,"context_line":"            self.frag_buf +\u003d chunk"},{"line_number":2659,"context_line":"            cursor \u003d 0"},{"line_number":2660,"context_line":"            while len(self.frag_buf) \u003e\u003d cursor + self.frag_size:"},{"line_number":2661,"context_line":"                self._check_frag(self.frag_buf[cursor:cursor + self.frag_size])"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_d70d683f","line":2658,"updated":"2016-11-01 01:09:07.000000000","message":"What are the relative sizes of len(chunk) and self.frag_size, typically?","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"06da9a6137027e0fc03cab9245044b4c778f71b4","unresolved":false,"context_lines":[{"line_number":2655,"context_line":"    def _update_checks(self, chunk):"},{"line_number":2656,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"},{"line_number":2657,"context_line":"        if self.frag_buf is not None:"},{"line_number":2658,"context_line":"            self.frag_buf +\u003d chunk"},{"line_number":2659,"context_line":"            cursor \u003d 0"},{"line_number":2660,"context_line":"            while len(self.frag_buf) \u003e\u003d cursor + self.frag_size:"},{"line_number":2661,"context_line":"                self._check_frag(self.frag_buf[cursor:cursor + self.frag_size])"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_5dd23d5d","line":2658,"in_reply_to":"ba5da102_bae0bffb","updated":"2016-11-01 18:20:45.000000000","message":"I don\u0027t know what you mean by \"quadratic behavior\" - it sounds bad tho?\n\nThere\u0027s probably some quick math to figure it out fragment_size but I tested with a couple of policies (had to create rings)\n\n    default saio 4+2 \u003d\u003e fragment_size \u003d 262224\n    typical prod 8+2 \u003d\u003e fragment_size \u003d 131152\n\nso that\u0027s roughly what we\u0027re dealing with - as you spread your default 1M segment size across more frags they get smaller.  I\u0027ve seen some evidence that some deployments might be trending towards larger segment sizes tho.\n\nIn all cases I know about people stick with the default chunk size in the object-server which is 65536\n\nIn the auditor I think it\u0027s not uncommon to do large io\u0027s with some sort of ratelimit ionice stuff in place - there the chunksize might be 256K or more?\n\n... so ... should the code change before we start quarantining invalid fragments?","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"8aa4c70611733955bf5629ac67b5f33b54e75a07","unresolved":false,"context_lines":[{"line_number":2655,"context_line":"    def _update_checks(self, chunk):"},{"line_number":2656,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"},{"line_number":2657,"context_line":"        if self.frag_buf is not None:"},{"line_number":2658,"context_line":"            self.frag_buf +\u003d chunk"},{"line_number":2659,"context_line":"            cursor \u003d 0"},{"line_number":2660,"context_line":"            while len(self.frag_buf) \u003e\u003d cursor + self.frag_size:"},{"line_number":2661,"context_line":"                self._check_frag(self.frag_buf[cursor:cursor + self.frag_size])"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_dd64a9e6","line":2658,"in_reply_to":"ba5da102_d70d683f","updated":"2016-11-01 02:29:35.000000000","message":"I don\u0027t think so right now. The self.frag_size is the relative value with segment size and ec_k but chunk size is related to disk_chunk_size in the object-server.conf. We might define to static value to fit the disk_chunk_size with fragment size if we uses only a policy but on the other hand if using more than one policy and each policy is using different parameter for EC (or Repl policy) it may be hard to make it fixed value.","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ce02246903c8b712255095e663f3d7d1de192648","unresolved":false,"context_lines":[{"line_number":2655,"context_line":"    def _update_checks(self, chunk):"},{"line_number":2656,"context_line":"        super(ECDiskFileReader, self)._update_checks(chunk)"},{"line_number":2657,"context_line":"        if self.frag_buf is not None:"},{"line_number":2658,"context_line":"            self.frag_buf +\u003d chunk"},{"line_number":2659,"context_line":"            cursor \u003d 0"},{"line_number":2660,"context_line":"            while len(self.frag_buf) \u003e\u003d cursor + self.frag_size:"},{"line_number":2661,"context_line":"                self._check_frag(self.frag_buf[cursor:cursor + self.frag_size])"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_bae0bffb","line":2658,"in_reply_to":"ba5da102_dd64a9e6","updated":"2016-11-01 17:55:45.000000000","message":"I ask because all this cursor business seems to indicate that we expect self.frag_size \u003c\u003c len(chunk) as it prevents quadratic behavior in the easier-to-reason-about\n\n while len(self.frag_buf) \u003e\u003d self.frag_size:\n     self._check_frag(self.frag_buf[:self.frag_size])\n     self.frag_buf \u003d self.frag_buf[self.frag_size:]\n\nIf this is generally the case (or even if self.frag_size and len(chunk) are approximately the same size), great! But if self.frag_size can be \u003e\u003e len(chunk), then\n\n self.frag_buf +\u003d chunk\n\nwill have quadratic behavior.","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"}],"test/unit/obj/common.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d17f1444f7ce65773002f38731213f61959146d0","unresolved":false,"context_lines":[{"line_number":98,"context_line":"                       commit\u003dcommit)"},{"line_number":99,"context_line":"        if commit:"},{"line_number":100,"context_line":"            # when we write and commit stub data, sanity check it\u0027s readable"},{"line_number":101,"context_line":"            # and not quaratined because of any validation check"},{"line_number":102,"context_line":"            with df.open():"},{"line_number":103,"context_line":"                self.assertEqual(\u0027\u0027.join(df.reader()), body)"},{"line_number":104,"context_line":"            # sanity checks"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_a9bb8971","line":101,"range":{"start_line":101,"start_character":22,"end_line":101,"end_character":32},"updated":"2016-10-21 15:19:38.000000000","message":"typo: quarantined","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5bd84e2413bd21cb255505dbdc7ed019c7e9761e","unresolved":false,"context_lines":[{"line_number":98,"context_line":"                       commit\u003dcommit)"},{"line_number":99,"context_line":"        if commit:"},{"line_number":100,"context_line":"            # when we write and commit stub data, sanity check it\u0027s readable"},{"line_number":101,"context_line":"            # and not quaratined because of any validation check"},{"line_number":102,"context_line":"            with df.open():"},{"line_number":103,"context_line":"                self.assertEqual(\u0027\u0027.join(df.reader()), body)"},{"line_number":104,"context_line":"            # sanity checks"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_64bfd8d0","line":101,"range":{"start_line":101,"start_character":22,"end_line":101,"end_character":32},"in_reply_to":"da6895a0_a9bb8971","updated":"2016-10-21 15:30:06.000000000","message":"Done","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"}],"test/unit/obj/test_auditor.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"733553563f6c7a49840775244fb1f6fc70cd67e5","unresolved":false,"context_lines":[{"line_number":156,"context_line":"                                                   self.rcache, self.devices)"},{"line_number":157,"context_line":"            data \u003d b\u00270\u0027 * 1024"},{"line_number":158,"context_line":"            if disk_file.policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":159,"context_line":"                data \u003d disk_file.policy.pyeclib_driver.encode(data)[0]"},{"line_number":160,"context_line":"            etag \u003d md5()"},{"line_number":161,"context_line":"            with disk_file.create() as writer:"},{"line_number":162,"context_line":"                writer.write(data)"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_0bd71f24","line":159,"updated":"2016-10-17 19:59:05.000000000","message":"I bet there\u0027s a ton of places where this *should* happen and a helper method may be needed. Tests pass, but are they testing the intended quarantine behaviour now??","commit_id":"ddadb1a07b7a350de6087a8f9421cf957d907cf4"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":3118,"context_line":"        quarantine_msgs \u003d []"},{"line_number":3119,"context_line":"        reader \u003d df.reader(_quarantine_hook\u003dquarantine_msgs.append)"},{"line_number":3120,"context_line":"        obj_size \u003d reader._obj_size"},{"line_number":3121,"context_line":"        self.assertEqual(obj_size, reader._obj_size)  # sanity check"},{"line_number":3122,"context_line":"        reader._obj_size +\u003d 1"},{"line_number":3123,"context_line":"        it \u003d reader.app_iter_ranges([(0, obj_size)],"},{"line_number":3124,"context_line":"                                    \u0027plain/text\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_df73184c","line":3121,"updated":"2016-10-21 12:15:46.000000000","message":"unnecessary sanity check - fixed","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":3217,"context_line":"                                       frag_index\u003dfrag_index)"},{"line_number":3218,"context_line":"        data \u003d data or \u00270\u0027 * fsize"},{"line_number":3219,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3220,"context_line":"            archive \u003d []"},{"line_number":3221,"context_line":"            while data:"},{"line_number":3222,"context_line":"                frags \u003d policy.pyeclib_driver.encode("},{"line_number":3223,"context_line":"                    data[:policy.ec_segment_size])"},{"line_number":3224,"context_line":"                try:"},{"line_number":3225,"context_line":"                    frag \u003d frags[df._frag_index]"},{"line_number":3226,"context_line":"                except IndexError:"},{"line_number":3227,"context_line":"                    # when a test passes in an invalid frag_index the *real*"},{"line_number":3228,"context_line":"                    # frag index is buried in the metadata somewhere"},{"line_number":3229,"context_line":"                    frag \u003d frags[0]"},{"line_number":3230,"context_line":"                archive.append(frag)"},{"line_number":3231,"context_line":"                data \u003d data[policy.ec_segment_size:]"},{"line_number":3232,"context_line":"            data \u003d \u0027\u0027.join(archive)"},{"line_number":3233,"context_line":"        etag \u003d md5()"},{"line_number":3234,"context_line":"        if ts:"},{"line_number":3235,"context_line":"            timestamp \u003d Timestamp(ts)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_5087040f","line":3232,"range":{"start_line":3220,"start_character":12,"end_line":3232,"end_character":35},"updated":"2016-10-21 08:50:10.000000000","message":"nits) it looks like we can use test.unit.encode_frag_archive_bodies function here?\n\nOnce I replaced the code with:\n\n  archives \u003d encode_frag_archive_bodies(policy, data)\n  try:\n    data \u003d archives[df._frag_index]\n  except IndexError:\n    data \u003d archives[0]\n\nNothing failed :D","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":3217,"context_line":"                                       frag_index\u003dfrag_index)"},{"line_number":3218,"context_line":"        data \u003d data or \u00270\u0027 * fsize"},{"line_number":3219,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3220,"context_line":"            archive \u003d []"},{"line_number":3221,"context_line":"            while data:"},{"line_number":3222,"context_line":"                frags \u003d policy.pyeclib_driver.encode("},{"line_number":3223,"context_line":"                    data[:policy.ec_segment_size])"},{"line_number":3224,"context_line":"                try:"},{"line_number":3225,"context_line":"                    frag \u003d frags[df._frag_index]"},{"line_number":3226,"context_line":"                except IndexError:"},{"line_number":3227,"context_line":"                    # when a test passes in an invalid frag_index the *real*"},{"line_number":3228,"context_line":"                    # frag index is buried in the metadata somewhere"},{"line_number":3229,"context_line":"                    frag \u003d frags[0]"},{"line_number":3230,"context_line":"                archive.append(frag)"},{"line_number":3231,"context_line":"                data \u003d data[policy.ec_segment_size:]"},{"line_number":3232,"context_line":"            data \u003d \u0027\u0027.join(archive)"},{"line_number":3233,"context_line":"        etag \u003d md5()"},{"line_number":3234,"context_line":"        if ts:"},{"line_number":3235,"context_line":"            timestamp \u003d Timestamp(ts)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_df55d81d","line":3232,"range":{"start_line":3220,"start_character":12,"end_line":3232,"end_character":35},"in_reply_to":"da6895a0_5087040f","updated":"2016-10-21 12:15:46.000000000","message":"Done by kota","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":4440,"context_line":"            with mock.patch(\"swift.obj.diskfile.DROP_CACHE_WINDOW\", 4095):"},{"line_number":4441,"context_line":"                with open(\u0027/dev/null\u0027, \u0027w\u0027) as devnull:"},{"line_number":4442,"context_line":"                    reader.zero_copy_send(devnull.fileno())"},{"line_number":4443,"context_line":"                if df.policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":4444,"context_line":"                    expected \u003d 4 + 1"},{"line_number":4445,"context_line":"                else:"},{"line_number":4446,"context_line":"                    expected \u003d (4 * 10) + 1"},{"line_number":4447,"context_line":"                self.assertEqual(len(dbc.mock_calls), expected)"},{"line_number":4448,"context_line":""},{"line_number":4449,"context_line":"    def test_zero_copy_turns_off_when_md5_sockets_not_supported(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_3bb991ea","line":4446,"range":{"start_line":4443,"start_character":14,"end_line":4446,"end_character":43},"updated":"2016-10-21 08:50:10.000000000","message":"nits) probably this is true becaus 163840/4095 is 40.0097680098 so 40 + 1 times called in the replication case. And the ec_default policy for the unit test is configured ec_ndata \u003d 10 which means each fragment archive keeps the encoded content body *about* 1/10 of the original body.\n\nThe reason I said as \"about\" because each fragments in the fragment archive containers fragment header so either if segment size is smaller enough or if file size is bigger enough (which result in sum of the fragment header overhead become bigger than a fragment), the condition may not be directly intuitive.\n\nIt\u0027s ok this is as it is or fix in follow up. I like to make it calculated from the actual file size. (Sorry, no example for now)","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":4440,"context_line":"            with mock.patch(\"swift.obj.diskfile.DROP_CACHE_WINDOW\", 4095):"},{"line_number":4441,"context_line":"                with open(\u0027/dev/null\u0027, \u0027w\u0027) as devnull:"},{"line_number":4442,"context_line":"                    reader.zero_copy_send(devnull.fileno())"},{"line_number":4443,"context_line":"                if df.policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":4444,"context_line":"                    expected \u003d 4 + 1"},{"line_number":4445,"context_line":"                else:"},{"line_number":4446,"context_line":"                    expected \u003d (4 * 10) + 1"},{"line_number":4447,"context_line":"                self.assertEqual(len(dbc.mock_calls), expected)"},{"line_number":4448,"context_line":""},{"line_number":4449,"context_line":"    def test_zero_copy_turns_off_when_md5_sockets_not_supported(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_7f25acc0","line":4446,"range":{"start_line":4443,"start_character":14,"end_line":4446,"end_character":43},"in_reply_to":"da6895a0_3bb991ea","updated":"2016-10-21 12:15:46.000000000","message":"follow up","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":5476,"context_line":"            it \u003d reader.app_iter_ranges("},{"line_number":5477,"context_line":"                [(0, 10), (10, 20),"},{"line_number":5478,"context_line":"                 (frag_size + 20, frag_size + 30)],"},{"line_number":5479,"context_line":"                \u0027plain/text\u0027, \u0027\\r\\n--someheader\\r\\n\u0027, 30)"},{"line_number":5480,"context_line":"            value \u003d \u0027\u0027.join(it)"},{"line_number":5481,"context_line":"        # check that only first range which starts at 0 triggers a frag check"},{"line_number":5482,"context_line":"        self.assertEqual(1, mock_get_metadata.call_count)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_2aabfd8a","line":5479,"range":{"start_line":5479,"start_character":53,"end_line":5479,"end_character":56},"updated":"2016-10-21 08:50:10.000000000","message":"small nits)\nthe last arg for app_iter_ranges is \"size\" which is from content_length and it results in content-range header like\n\n Content-Range: 0-10/\u003csize\u003e\n\nHence, I think\n\n it \u003d reader.app_iter_ranges(\n    [(0, 10), (10, 20),\n     (frag_size + 20, frag_size + 30)],\n     \u0027plain/text\u0027, \u0027\\r\\n--someheader\\r\\n\u0027, len(df_data))\n\nis better.\n\n...Oops, this is from other tests and others are also using bad ones. I think it should be in separate patch as follow up to fix them.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":5476,"context_line":"            it \u003d reader.app_iter_ranges("},{"line_number":5477,"context_line":"                [(0, 10), (10, 20),"},{"line_number":5478,"context_line":"                 (frag_size + 20, frag_size + 30)],"},{"line_number":5479,"context_line":"                \u0027plain/text\u0027, \u0027\\r\\n--someheader\\r\\n\u0027, 30)"},{"line_number":5480,"context_line":"            value \u003d \u0027\u0027.join(it)"},{"line_number":5481,"context_line":"        # check that only first range which starts at 0 triggers a frag check"},{"line_number":5482,"context_line":"        self.assertEqual(1, mock_get_metadata.call_count)"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_1f1d9073","line":5479,"range":{"start_line":5479,"start_character":53,"end_line":5479,"end_character":56},"in_reply_to":"da6895a0_2aabfd8a","updated":"2016-10-21 12:15:46.000000000","message":"Done","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d17f1444f7ce65773002f38731213f61959146d0","unresolved":false,"context_lines":[{"line_number":2842,"context_line":"                frag \u003d frags[df._frag_index]"},{"line_number":2843,"context_line":"                archive.append(frag)"},{"line_number":2844,"context_line":"                data \u003d data[df.policy.ec_segment_size:]"},{"line_number":2845,"context_line":"            data \u003d \u0027\u0027.join(archive)"},{"line_number":2846,"context_line":""},{"line_number":2847,"context_line":"        with df.create() as writer:"},{"line_number":2848,"context_line":"            new_metadata \u003d {"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_a9ec69c8","line":2845,"updated":"2016-10-21 15:19:38.000000000","message":"we can use the helper method here, encode_frag_archive_bodies","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5bd84e2413bd21cb255505dbdc7ed019c7e9761e","unresolved":false,"context_lines":[{"line_number":2842,"context_line":"                frag \u003d frags[df._frag_index]"},{"line_number":2843,"context_line":"                archive.append(frag)"},{"line_number":2844,"context_line":"                data \u003d data[df.policy.ec_segment_size:]"},{"line_number":2845,"context_line":"            data \u003d \u0027\u0027.join(archive)"},{"line_number":2846,"context_line":""},{"line_number":2847,"context_line":"        with df.create() as writer:"},{"line_number":2848,"context_line":"            new_metadata \u003d {"}],"source_content_type":"text/x-python","patch_set":9,"id":"da6895a0_64d8b854","line":2845,"in_reply_to":"da6895a0_a9ec69c8","updated":"2016-10-21 15:30:06.000000000","message":"Done","commit_id":"5a9d423e30e5266518b90a2309c4c2f59599825c"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":3293,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3294,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3295,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3296,"context_line":"            body \u003d policy.pyeclib_driver.encode(body)[0]"},{"line_number":3297,"context_line":"            headers.update({"},{"line_number":3298,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00270\u0027,"},{"line_number":3299,"context_line":"                \u0027Content-Length\u0027: len(body),"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_7b74e9ce","line":3296,"updated":"2016-10-21 08:50:10.000000000","message":"nits) it might be useful of encode_frag_archive_bodies here as well as a comment i suggested in other place.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":3293,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3294,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3295,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3296,"context_line":"            body \u003d policy.pyeclib_driver.encode(body)[0]"},{"line_number":3297,"context_line":"            headers.update({"},{"line_number":3298,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00270\u0027,"},{"line_number":3299,"context_line":"                \u0027Content-Length\u0027: len(body),"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_5a5d56e8","line":3296,"in_reply_to":"da6895a0_7b74e9ce","updated":"2016-10-21 12:15:46.000000000","message":"Done","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":3324,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3325,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3326,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3327,"context_line":"            body \u003d policy.pyeclib_driver.encode(body)[2]"},{"line_number":3328,"context_line":"            headers.update({"},{"line_number":3329,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00272\u0027,"},{"line_number":3330,"context_line":"                \u0027Content-Length\u0027: len(body),"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_bb7e81ac","line":3327,"updated":"2016-10-21 08:50:10.000000000","message":"ditto","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":3324,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3325,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3326,"context_line":"        if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3327,"context_line":"            body \u003d policy.pyeclib_driver.encode(body)[2]"},{"line_number":3328,"context_line":"            headers.update({"},{"line_number":3329,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00272\u0027,"},{"line_number":3330,"context_line":"                \u0027Content-Length\u0027: len(body),"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_5a1476f4","line":3327,"in_reply_to":"da6895a0_bb7e81ac","updated":"2016-10-21 12:15:46.000000000","message":"Done","commit_id":"8ea029d86118100613d111d662399cb5575342bb"}],"test/unit/obj/test_ssync.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d194466bf520dda54067797b2ec1e764c5186cff","unresolved":false,"context_lines":[{"line_number":123,"context_line":"            self.obj_data[path] \u003d \u0027%s___data\u0027 % path"},{"line_number":124,"context_line":"        return self.obj_data[path]"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"    def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp,"},{"line_number":127,"context_line":"                             frag_indexes\u003dNone, commit\u003dTrue):"},{"line_number":128,"context_line":"        frag_indexes \u003d [None] if not frag_indexes else frag_indexes"},{"line_number":129,"context_line":"        metadata \u003d {\u0027Content-Type\u0027: \u0027plain/text\u0027}"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa6399be_6015b499","line":126,"updated":"2016-10-19 19:38:47.000000000","message":"this method could be subclass specific (EC vs replica) but works for now","commit_id":"3c35334bf5180ad53c9d452c338c8769e1abe079"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d194466bf520dda54067797b2ec1e764c5186cff","unresolved":false,"context_lines":[{"line_number":315,"context_line":"        # return a frag archive for given object name and frag index."},{"line_number":316,"context_line":"        # for EC policies obj_data maps obj path -\u003e list of frag archives"},{"line_number":317,"context_line":"        if path not in self.obj_data:"},{"line_number":318,"context_line":"            # make unique frag archives for each object name"},{"line_number":319,"context_line":"            data \u003d path * 2 * (self.policy.ec_ndata + self.policy.ec_nparity)"},{"line_number":320,"context_line":"            self.obj_data[path] \u003d encode_frag_archive_bodies("},{"line_number":321,"context_line":"                self.policy, data)"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa6399be_e021c4fd","line":318,"updated":"2016-10-19 19:38:47.000000000","message":"note the data has to be long enough and unique enough that ALL frags will differ between two objects\n\n2 * total frags seems safe","commit_id":"3c35334bf5180ad53c9d452c338c8769e1abe079"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":136,"context_line":"                listing \u003d os.listdir(df._datadir)"},{"line_number":137,"context_line":"                self.assertTrue(listing)"},{"line_number":138,"context_line":"                for filename in listing:"},{"line_number":139,"context_line":"                    self.assertTrue(filename.startswith(timestamp.internal))"},{"line_number":140,"context_line":"            diskfiles.append(df)"},{"line_number":141,"context_line":"        return diskfiles"},{"line_number":142,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_b65f780d","side":"PARENT","line":139,"updated":"2016-10-21 08:50:10.000000000","message":"OK, commit was moved into _make_diskfile.","commit_id":"a79d8508df493d5744b262e1d1830782e32dbd04"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"    def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp,"},{"line_number":127,"context_line":"                             frag_indexes\u003dNone, commit\u003dTrue):"},{"line_number":128,"context_line":"        frag_indexes \u003d [None] if not frag_indexes else frag_indexes"},{"line_number":129,"context_line":"        metadata \u003d {\u0027Content-Type\u0027: \u0027plain/text\u0027}"},{"line_number":130,"context_line":"        diskfiles \u003d []"},{"line_number":131,"context_line":"        for frag_index in frag_indexes:"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_b648585d","line":128,"range":{"start_line":128,"start_character":7,"end_line":128,"end_character":67},"updated":"2016-10-21 08:50:10.000000000","message":"nits) I like:\n\n\n frag_indexes \u003d frag_indexes or [None]\n\nfor simplify.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"baae2b8e68212ca78666f381a00c32081ef1474e","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"    def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp,"},{"line_number":127,"context_line":"                             frag_indexes\u003dNone, commit\u003dTrue):"},{"line_number":128,"context_line":"        frag_indexes \u003d [None] if not frag_indexes else frag_indexes"},{"line_number":129,"context_line":"        metadata \u003d {\u0027Content-Type\u0027: \u0027plain/text\u0027}"},{"line_number":130,"context_line":"        diskfiles \u003d []"},{"line_number":131,"context_line":"        for frag_index in frag_indexes:"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_9240a958","line":128,"in_reply_to":"da6895a0_6617e951","updated":"2016-10-21 19:53:19.000000000","message":"well but the original line seemed to want frag_indexes to be [] if that was what was passed in explicitly?\n\nI\u0027d be sort of annoyed if I called this helper with frag_indexes\u003d[] and got [None] instead right?","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"    def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp,"},{"line_number":127,"context_line":"                             frag_indexes\u003dNone, commit\u003dTrue):"},{"line_number":128,"context_line":"        frag_indexes \u003d [None] if not frag_indexes else frag_indexes"},{"line_number":129,"context_line":"        metadata \u003d {\u0027Content-Type\u0027: \u0027plain/text\u0027}"},{"line_number":130,"context_line":"        diskfiles \u003d []"},{"line_number":131,"context_line":"        for frag_index in frag_indexes:"}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_6617e951","line":128,"range":{"start_line":128,"start_character":7,"end_line":128,"end_character":67},"in_reply_to":"da6895a0_b648585d","updated":"2016-10-21 12:15:46.000000000","message":"I like that too!","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"ecc02aa6c0287fd2f53d9e156f2f7a4d28873290","unresolved":false,"context_lines":[{"line_number":772,"context_line":"    def getheaders(self):"},{"line_number":773,"context_line":"        return self.headers"},{"line_number":774,"context_line":""},{"line_number":775,"context_line":"    def read(self, length):"},{"line_number":776,"context_line":"        if isinstance(length, Exception):"},{"line_number":777,"context_line":"            raise self.length"},{"line_number":778,"context_line":"        val \u003d self.data"},{"line_number":779,"context_line":"        self.data \u003d \u0027\u0027"},{"line_number":780,"context_line":"        return val if self.length is None else val[:self.length]"},{"line_number":781,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_361448f4","line":778,"range":{"start_line":775,"start_character":3,"end_line":778,"end_character":5},"updated":"2016-10-21 08:50:10.000000000","message":"*Is length a kind of Exception!?!?*\n\nThis means reconstructor can feed an exception instance to read method?\n\nOr just like:\n\n if isinstance(self.length, Exception)\n\n???\n\nEither case, it sounds odd though.\n\nAh, that\u0027s because self.data is now used for encoded object. hmmm... I\u0027m feeling we should consider the impl at init method.\n\nhttp://paste.openstack.org/show/586662/ works probably as you expected.","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4c023ca47f0dbf1f8ce63fbcd41c4a14e7aee906","unresolved":false,"context_lines":[{"line_number":772,"context_line":"    def getheaders(self):"},{"line_number":773,"context_line":"        return self.headers"},{"line_number":774,"context_line":""},{"line_number":775,"context_line":"    def read(self, length):"},{"line_number":776,"context_line":"        if isinstance(length, Exception):"},{"line_number":777,"context_line":"            raise self.length"},{"line_number":778,"context_line":"        val \u003d self.data"},{"line_number":779,"context_line":"        self.data \u003d \u0027\u0027"},{"line_number":780,"context_line":"        return val if self.length is None else val[:self.length]"},{"line_number":781,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_9a1d2e12","line":778,"range":{"start_line":775,"start_character":3,"end_line":778,"end_character":5},"in_reply_to":"da6895a0_361448f4","updated":"2016-10-21 12:15:46.000000000","message":"Done applied Kota\u0027s fix","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"baae2b8e68212ca78666f381a00c32081ef1474e","unresolved":false,"context_lines":[{"line_number":775,"context_line":"    def read(self, length):"},{"line_number":776,"context_line":"        if isinstance(length, Exception):"},{"line_number":777,"context_line":"            raise self.length"},{"line_number":778,"context_line":"        val \u003d self.data"},{"line_number":779,"context_line":"        self.data \u003d \u0027\u0027"},{"line_number":780,"context_line":"        return val if self.length is None else val[:self.length]"},{"line_number":781,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"da6895a0_12049915","line":778,"in_reply_to":"da6895a0_9a1d2e12","updated":"2016-10-21 19:53:19.000000000","message":"no idea what is going on here - Kota\u0027s paste looks good","commit_id":"8ea029d86118100613d111d662399cb5575342bb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d0a32cc483ec29f8755c3bde1826ba5f364bb617","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"    def _create_ondisk_files(self, df_mgr, obj_name, policy, timestamp,"},{"line_number":127,"context_line":"                             frag_indexes\u003dNone, commit\u003dTrue):"},{"line_number":128,"context_line":"        frag_indexes \u003d frag_indexes or [None]"},{"line_number":129,"context_line":"        metadata \u003d {\u0027Content-Type\u0027: \u0027plain/text\u0027}"},{"line_number":130,"context_line":"        diskfiles \u003d []"},{"line_number":131,"context_line":"        for frag_index in frag_indexes:"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_2ebab10a","line":128,"updated":"2016-11-01 20:10:55.000000000","message":"ok, so either you pass in frag_indexes or you\u0027re replicated, frag_indexes\u003d[] just isn\u0027t a thing.","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d0a32cc483ec29f8755c3bde1826ba5f364bb617","unresolved":false,"context_lines":[{"line_number":770,"context_line":"        if isinstance(self.obj_data, Exception):"},{"line_number":771,"context_line":"            self.data \u003d self.obj_data"},{"line_number":772,"context_line":"        else:"},{"line_number":773,"context_line":"            self.data \u003d self.obj_data[path][self.frag_index]"},{"line_number":774,"context_line":""},{"line_number":775,"context_line":"    def getheaders(self):"},{"line_number":776,"context_line":"        return self.headers"}],"source_content_type":"text/x-python","patch_set":11,"id":"ba5da102_e34d323f","line":773,"updated":"2016-11-01 20:10:55.000000000","message":"I don\u0027t love having a __init__ and init methods on this Fake, but I can\u0027t come up with anything that cleans this up in a material way.\n\nThe problem is that we need to setup the tests with sets of responses in a specific order \"this set of specific failures, followed by a reasonable set of responses\" to verify the whole things blows up in a predictable way - but the ssync_sender doesn\u0027t really cooperate - the order that it *asks* for responses is based on the dict key orders of the send_map it builds up during missing_check\n\nI started to think maybe FakeResponses shouldn\u0027t have a body at all - and somewhere in fake_get_response we back fill the bodies of the fake_responses from our self.obj_data but ... well that\u0027s sort of what\u0027s going on now with maybe a little teeny tiny bit less passing/bleeding of state ... and then it doesn\u0027t really feel like a worth while material clean up.\n\nSorta six-one-way still a pile-of-mess the other.","commit_id":"854d25d4862a683e5264dfe28a2e5caa908991a4"}]}