)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":22,"context_line":"will label it with an ETag that matches the md5 sum of the bad"},{"line_number":23,"context_line":"data. The bad fragment file will therefore appear good to the auditor."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"There is no easy way for the ssync sender to communicate a lack of"},{"line_number":26,"context_line":"source data to the receiver other than by disconnecting the"},{"line_number":27,"context_line":"session. So this patch adds a check in the ssync sender that the sent"},{"line_number":28,"context_line":"byte count is equal to the sent Content-Length header value for each"},{"line_number":29,"context_line":"subrequest, and disconnect if a mismatch is detected."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The disconnect prevents the receiver finalizing the bad diskfile, but"},{"line_number":32,"context_line":"also prevents subsequent fragments in the ssync job being sync\u0027d until"},{"line_number":33,"context_line":"the next cycle."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Partial-Bug: #1631144"},{"line_number":36,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"fa6399be_601cd477","line":33,"range":{"start_line":25,"start_character":0,"end_line":33,"end_character":15},"updated":"2016-10-13 08:50:41.000000000","message":"Yeah, this is what I was thinking last night until I read the clay\u0027s comment. Since his comment, I\u0027m realizing we could make the ssync_sender can notify the lack of body to ssync_receiver via sending empty chunk. I\u0027ll try it.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":22,"context_line":"will label it with an ETag that matches the md5 sum of the bad"},{"line_number":23,"context_line":"data. The bad fragment file will therefore appear good to the auditor."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"There is no easy way for the ssync sender to communicate a lack of"},{"line_number":26,"context_line":"source data to the receiver other than by disconnecting the"},{"line_number":27,"context_line":"session. So this patch adds a check in the ssync sender that the sent"},{"line_number":28,"context_line":"byte count is equal to the sent Content-Length header value for each"},{"line_number":29,"context_line":"subrequest, and disconnect if a mismatch is detected."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The disconnect prevents the receiver finalizing the bad diskfile, but"},{"line_number":32,"context_line":"also prevents subsequent fragments in the ssync job being sync\u0027d until"},{"line_number":33,"context_line":"the next cycle."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Partial-Bug: #1631144"},{"line_number":36,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"fa6399be_d8049f4b","line":33,"range":{"start_line":25,"start_character":0,"end_line":33,"end_character":15},"in_reply_to":"fa6399be_601cd477","updated":"2016-10-13 14:26:39.000000000","message":"disconnect() method in ssync.Sender *does* send an empty chunk, and that signals end of chunked transfer. AFAICT it\u0027s not possible to send an empty chunk and then continue with more data transfer.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"}],"swift/obj/reconstructor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"73b07e65c5e0a455876dd8ac018ab1be62a41a23","unresolved":false,"context_lines":[{"line_number":78,"context_line":""},{"line_number":79,"context_line":"        # the new FA is going to have the same length as others in the set"},{"line_number":80,"context_line":"        self._content_length \u003d int(self.datafile_metadata[\u0027Content-Length\u0027])"},{"line_number":81,"context_line":"        self.bytes_read \u003d 0"},{"line_number":82,"context_line":"        # update the FI and delete the ETag, the obj server will"},{"line_number":83,"context_line":"        # recalc on the other side..."},{"line_number":84,"context_line":"        self.datafile_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027] \u003d frag_index"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_b5455018","line":81,"updated":"2016-10-13 01:41:37.000000000","message":"i don\u0027t think this state is used","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"20c049ce31061ec77737dbb7f5cccd20cbfa6949","unresolved":false,"context_lines":[{"line_number":78,"context_line":""},{"line_number":79,"context_line":"        # the new FA is going to have the same length as others in the set"},{"line_number":80,"context_line":"        self._content_length \u003d int(self.datafile_metadata[\u0027Content-Length\u0027])"},{"line_number":81,"context_line":"        self.bytes_read \u003d 0"},{"line_number":82,"context_line":"        # update the FI and delete the ETag, the obj server will"},{"line_number":83,"context_line":"        # recalc on the other side..."},{"line_number":84,"context_line":"        self.datafile_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027] \u003d frag_index"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_f4ef8e84","line":81,"in_reply_to":"fa6399be_986b07f1","updated":"2016-10-14 06:40:42.000000000","message":"Done","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":78,"context_line":""},{"line_number":79,"context_line":"        # the new FA is going to have the same length as others in the set"},{"line_number":80,"context_line":"        self._content_length \u003d int(self.datafile_metadata[\u0027Content-Length\u0027])"},{"line_number":81,"context_line":"        self.bytes_read \u003d 0"},{"line_number":82,"context_line":"        # update the FI and delete the ETag, the obj server will"},{"line_number":83,"context_line":"        # recalc on the other side..."},{"line_number":84,"context_line":"        self.datafile_metadata[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027] \u003d frag_index"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_986b07f1","line":81,"in_reply_to":"fa6399be_b5455018","updated":"2016-10-13 14:26:39.000000000","message":"my bad, at one point I was checking the sent bytes length in this class but decided it belonged in ssync sender since it was a check that *ssync* cares about and should be made irrespective of the particular diskfile implementation.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"32933da7c5a242207f99ad1699f9ff6edf877c36","unresolved":false,"context_lines":[{"line_number":99,"context_line":""},{"line_number":100,"context_line":"    def reader(self):"},{"line_number":101,"context_line":"        for chunk in self.rebuilt_fragment_iter:"},{"line_number":102,"context_line":"            yield chunk"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"class ObjectReconstructor(Daemon):"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_8f8c57db","line":102,"updated":"2016-10-14 22:33:56.000000000","message":"Since we setting _content_length on this object I spent a lot of time considering if the byte count check should go here - but ultimately I kept coming back to sending the required content-length bytes or blowing up being a protocol level concern.\n\nI still think our diskfile\u0027s should do a better job of indicating an error if they need to terminate early - but finally decided ssync should protect it\u0027s protocol no matter what stupid code we write in our diskfiles.","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"82f00a99a2a968c86e9d1947a7912d6c5d2ec4e3","unresolved":false,"context_lines":[{"line_number":331,"context_line":"                         \u0027frag_index\u0027: frag_index,"},{"line_number":332,"context_line":"                         })"},{"line_number":333,"context_line":"                    break"},{"line_number":334,"context_line":"                if not all(fragment_payload):"},{"line_number":335,"context_line":"                    break"},{"line_number":336,"context_line":"                rebuilt_fragment \u003d self._reconstruct("},{"line_number":337,"context_line":"                    policy, fragment_payload, frag_index)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_3da9e990","line":334,"range":{"start_line":334,"start_character":16,"end_line":334,"end_character":45},"updated":"2016-10-14 07:34:14.000000000","message":"Just a note for the follow up:\nI\u0027m realizing this one might be\n\n if not any(fragment_payload):\n\nif the \u0027if\u0027 condition means \u0027raise StopIteration successfully\u0027. Exactly, the fragments in the response should be same length but... i\u0027m not sure if BufferredHTTPResponse.read could return less byte string in some cases like connection with object-sever killed.\n\nI mean\n\n\n [(10bytes), (10bytes), (0bytes due to connection killed, not sure if this happens)]\n\nit\u0027s sufficient for \"not all(fragment_payload)\" but it looks an error case for \"Error trying to rebuild\" at L327, maybe?","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"32933da7c5a242207f99ad1699f9ff6edf877c36","unresolved":false,"context_lines":[{"line_number":331,"context_line":"                         \u0027frag_index\u0027: frag_index,"},{"line_number":332,"context_line":"                         })"},{"line_number":333,"context_line":"                    break"},{"line_number":334,"context_line":"                if not all(fragment_payload):"},{"line_number":335,"context_line":"                    break"},{"line_number":336,"context_line":"                rebuilt_fragment \u003d self._reconstruct("},{"line_number":337,"context_line":"                    policy, fragment_payload, frag_index)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_0f50470a","line":334,"in_reply_to":"fa6399be_3da9e990","updated":"2016-10-14 22:33:56.000000000","message":"ALL of these\u0027 breaks are bad - this early termination is no way to indicate an Error.  It can\u0027t be a plain DiskFileError though because that wouldn\u0027t properly terminate the stream.","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"}],"swift/obj/ssync_receiver.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c3a7b71defec7452523485adc21e2b0a9aa18976","unresolved":false,"context_lines":[{"line_number":488,"context_line":"                    resp.status_int \u003d\u003d http.HTTP_NOT_FOUND:"},{"line_number":489,"context_line":"                successes +\u003d 1"},{"line_number":490,"context_line":"            else:"},{"line_number":491,"context_line":"                self.app.logger.warning("},{"line_number":492,"context_line":"                    \u0027ssync subrequest failed with %s: %s %s\u0027 %"},{"line_number":493,"context_line":"                    (resp.status_int, method, subreq.path))"},{"line_number":494,"context_line":"                failures +\u003d 1"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_b44c36c9","line":491,"range":{"start_line":491,"start_character":31,"end_line":491,"end_character":39},"updated":"2016-10-14 06:44:19.000000000","message":"ok, i like warning rather than my patch making error :-P","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"}],"swift/obj/ssync_sender.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"73b07e65c5e0a455876dd8ac018ab1be62a41a23","unresolved":false,"context_lines":[{"line_number":415,"context_line":"            with exceptions.MessageTimeout("},{"line_number":416,"context_line":"                    self.daemon.node_timeout, \u0027send_put chunk\u0027):"},{"line_number":417,"context_line":"                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(chunk), chunk))"},{"line_number":418,"context_line":"        if bytes_read !\u003d df.content_length:"},{"line_number":419,"context_line":"            # Since we may now have partial state on the receiver we have to"},{"line_number":420,"context_line":"            # prevent the receiver finalising what may well be a bad or"},{"line_number":421,"context_line":"            # partially written diskfile. Unfortunately we have no other option"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_3569408c","line":418,"updated":"2016-10-13 01:41:37.000000000","message":"there\u0027s a similar check on the recv side:\n\nhttps://github.com/openstack/swift/blob/98c5e10e274c6a3d98493821499aee4acfb3808e/swift/obj/ssync_receiver.py#L468\n\nI experimented with replacing this raise with just a send(\u00270\\r\\n\\r\\n\u0027) and the receiver detects that it got an incomplete transfer.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":415,"context_line":"            with exceptions.MessageTimeout("},{"line_number":416,"context_line":"                    self.daemon.node_timeout, \u0027send_put chunk\u0027):"},{"line_number":417,"context_line":"                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(chunk), chunk))"},{"line_number":418,"context_line":"        if bytes_read !\u003d df.content_length:"},{"line_number":419,"context_line":"            # Since we may now have partial state on the receiver we have to"},{"line_number":420,"context_line":"            # prevent the receiver finalising what may well be a bad or"},{"line_number":421,"context_line":"            # partially written diskfile. Unfortunately we have no other option"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_1819b722","line":418,"in_reply_to":"fa6399be_04195a79","updated":"2016-10-13 14:26:39.000000000","message":"I agree with Kota that the logging can be improved.\n\nAs far as when we terminate the chunked transfer:\n\nNote that raising the exception here causes \u00270\\r\\n\\r\\n\u0027 to be sent in disconnect() method - and the goal is to disconnect(), there is no way to do any more useful work on subsequent objects, because...\n\nAFAICT once the sender sends \u00270\\r\\n\\r\\n\u0027 (in fact, once it sends \u00270\u0027) the receiver will not be able to read anymore from the wsgi input because the \u00270\u0027 chunk length indicates end of transfer and it latches in wsgi. [1] That is consistent with the RFC for chunked transfer [2]\n\nAs far as the receiver tracebacks go, subsequent to sending a \u00270\\r\\n\\r\\n\u0027 (if for example running the tests in test_ssync) you see the ssync receiver raise an exception with:\n\n  error: [Errno 32] Broken pipe\n\nwhen trying to *send* to the socket.\n\nAFAICT that is when the receiver has completed handling updates, sees it had a failure and raises an HTTPInternalServerError which the object server tried to send as a response. I can\u0027t yet see what causes the broken pipe - although the \u00270\\r\\n\\r\\n\u0027 stops the receiver wsgi input from serving any more bytes to read I cannot see it close the connection, so is it closed on the sender\u0027s side? I think it must be.\n\nNow, if we make the change Clay suggests, the effect on receiver side is identical to my patch (raising the ReplcationException here causes disconnect() to be called which sends \u00270\\r\\n\\r\\n\u0027) - you see the same tracebacks on the receiver. However, on the sender side you\u0027ll now see a traceback for a broken pipe error when the sends tries to send the next object body. As I say I cannot yet find why, but I have to infer that sending the \u00270\\r\\n\\r\\n\u0027 causes the connection to close. IDK.\n\nSo, wherever the \u00270\\r\\n\\r\\n\u0027 is sent, the connection is closed and we can\u0027t send any more objects. IMO better to do it by raising the exception and ending up calling disconnect than doing it here and then triggering an exception when next object is sent.\n\n[1] https://github.com/eventlet/eventlet/blob/master/eventlet/wsgi.py#L165\n\n[2] https://tools.ietf.org/html/rfc7230#section-4.1","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"20c049ce31061ec77737dbb7f5cccd20cbfa6949","unresolved":false,"context_lines":[{"line_number":415,"context_line":"            with exceptions.MessageTimeout("},{"line_number":416,"context_line":"                    self.daemon.node_timeout, \u0027send_put chunk\u0027):"},{"line_number":417,"context_line":"                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(chunk), chunk))"},{"line_number":418,"context_line":"        if bytes_read !\u003d df.content_length:"},{"line_number":419,"context_line":"            # Since we may now have partial state on the receiver we have to"},{"line_number":420,"context_line":"            # prevent the receiver finalising what may well be a bad or"},{"line_number":421,"context_line":"            # partially written diskfile. Unfortunately we have no other option"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_37fca871","line":418,"in_reply_to":"fa6399be_1819b722","updated":"2016-10-14 06:40:42.000000000","message":"Good thought. I was missing this exception can trigger to call disconnect. And if we call send(\u00270\\r\\n\\r\\n\u0027) here, probably calling send(\u00270\\r\\n\\r\\n\u0027) in *disconnect* method hits the broken pipe. OK, we should do just raise Exception here.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1a58558e10aa4930735888d838aee6a4bcbe41ae","unresolved":false,"context_lines":[{"line_number":415,"context_line":"            with exceptions.MessageTimeout("},{"line_number":416,"context_line":"                    self.daemon.node_timeout, \u0027send_put chunk\u0027):"},{"line_number":417,"context_line":"                self.connection.send(\u0027%x\\r\\n%s\\r\\n\u0027 % (len(chunk), chunk))"},{"line_number":418,"context_line":"        if bytes_read !\u003d df.content_length:"},{"line_number":419,"context_line":"            # Since we may now have partial state on the receiver we have to"},{"line_number":420,"context_line":"            # prevent the receiver finalising what may well be a bad or"},{"line_number":421,"context_line":"            # partially written diskfile. Unfortunately we have no other option"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_04195a79","line":418,"in_reply_to":"fa6399be_3569408c","updated":"2016-10-13 10:58:16.000000000","message":"Good catch, Clay. With [1], i confirmed send(\u00270\\r\\n\\r\\n\u0027) can notify the incomplete transfer. However, it causes same result, we cannot continue the remaining sync job because send(\u00270\\r\\n\\r\\n\u0027) seems to trigger connection close to the ssync_receiver. When I attempt to do that, i got broken pipe for the next object updating. However, either way kill connection with raising ReplicationException or send(\u00270\\r\\n\\r\\n\u0027) makes not good error log on *receiver side* like \"ERROR __call__ error with PUT /dev/9/a/c/o1 \" which is meaningless, imho.\n\nIn [1], I tried to make it better to notice that ssync failed for the object because no enough data comming. How do you think of that?\n\n1: https://review.openstack.org/#/c/385886","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"32933da7c5a242207f99ad1699f9ff6edf877c36","unresolved":false,"context_lines":[{"line_number":423,"context_line":"            # multiphase PUTs like the proxy uses for EC we could send a bad"},{"line_number":424,"context_line":"            # etag in a footer of this subrequest, but that is not supported."},{"line_number":425,"context_line":"            raise exceptions.ReplicationException("},{"line_number":426,"context_line":"                \u0027Sent data length does not match content-length\u0027)"},{"line_number":427,"context_line":""},{"line_number":428,"context_line":"    def send_post(self, url_path, df):"},{"line_number":429,"context_line":"        metadata \u003d df.get_metafile_metadata()"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_4f815ffa","line":426,"updated":"2016-10-14 22:33:56.000000000","message":"This looks annoying defensive (gah, we can\u0027t trust our own iterators!) but with an eye towards backports I think it\u0027s 100% the best way to solve this issue.  And also *apparently* we *need* this sort of protection from ourselves - making sure we blow up if we can\u0027t send the content-length we promised is *definitely* a protocol level concern.\n\nBut I think we should fix the RebuildingECDiskFileStream too - which hopefully means in the future we never see this message.","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"}],"test/unit/obj/test_ssync.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":718,"context_line":"        # GET request for a fragment archive. Items in the outer list are lists"},{"line_number":719,"context_line":"        # of repsonses for each GET path. Items in the inner lists are"},{"line_number":720,"context_line":"        # responses for each frag index for a GET path."},{"line_number":721,"context_line":"        frag_responses \u003d list(frag_responses)"},{"line_number":722,"context_line":"        frag_responses.reverse()"},{"line_number":723,"context_line":"        path_to_responses \u003d {}"},{"line_number":724,"context_line":"        fake_get_response_calls \u003d []"},{"line_number":725,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_eca0daa1","line":722,"range":{"start_line":721,"start_character":0,"end_line":722,"end_character":32},"updated":"2016-10-13 08:50:41.000000000","message":"nits) it looks those 2 lines are redundant or we could take another way.\n\ni think, the first line is for safety if it\u0027s not a list but the inline comment above says \"frag_response is a list of list of responses\" so i\u0027m feeling redundant.\n\nthe next line looks like for the order of get_responses but I\u0027m feeling we could reduce the code with\n\n path_to_responses[path] \u003d frag_responses.pop(0)\n\nat L730.\n\nWhen I removes those 2 lines and then did like above at L730, all tests passed.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"20c049ce31061ec77737dbb7f5cccd20cbfa6949","unresolved":false,"context_lines":[{"line_number":718,"context_line":"        # GET request for a fragment archive. Items in the outer list are lists"},{"line_number":719,"context_line":"        # of repsonses for each GET path. Items in the inner lists are"},{"line_number":720,"context_line":"        # responses for each frag index for a GET path."},{"line_number":721,"context_line":"        frag_responses \u003d list(frag_responses)"},{"line_number":722,"context_line":"        frag_responses.reverse()"},{"line_number":723,"context_line":"        path_to_responses \u003d {}"},{"line_number":724,"context_line":"        fake_get_response_calls \u003d []"},{"line_number":725,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_748b7ec3","line":722,"range":{"start_line":721,"start_character":0,"end_line":722,"end_character":32},"in_reply_to":"fa6399be_9810675b","updated":"2016-10-14 06:40:42.000000000","message":"Done","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":718,"context_line":"        # GET request for a fragment archive. Items in the outer list are lists"},{"line_number":719,"context_line":"        # of repsonses for each GET path. Items in the inner lists are"},{"line_number":720,"context_line":"        # responses for each frag index for a GET path."},{"line_number":721,"context_line":"        frag_responses \u003d list(frag_responses)"},{"line_number":722,"context_line":"        frag_responses.reverse()"},{"line_number":723,"context_line":"        path_to_responses \u003d {}"},{"line_number":724,"context_line":"        fake_get_response_calls \u003d []"},{"line_number":725,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_9810675b","line":722,"range":{"start_line":721,"start_character":0,"end_line":722,"end_character":32},"in_reply_to":"fa6399be_eca0daa1","updated":"2016-10-13 14:26:39.000000000","message":"good call, and you understand the intent correctly.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":736,"context_line":"            # the reconstructor will try to remove the receiver node from the"},{"line_number":737,"context_line":"            # object ring part nodes, but the fake node we created for our"},{"line_number":738,"context_line":"            # receiver is not actually in the ring part nodes, so append it"},{"line_number":739,"context_line":"            # here simply so that the reconstructor does not fail to remove it."},{"line_number":740,"context_line":"            return (self.policy.object_ring._get_part_nodes(part) +"},{"line_number":741,"context_line":"                    [self.job_node])"},{"line_number":742,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_a926d4cb","line":739,"updated":"2016-10-13 08:50:41.000000000","message":"Oh, yeah. That is https://github.com/openstack/swift/blob/master/swift/obj/reconstructor.py#L242","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":741,"context_line":"                    [self.job_node])"},{"line_number":742,"context_line":""},{"line_number":743,"context_line":"        def fake_reconstruct(self, policy, fragment_payload, frag_index):"},{"line_number":744,"context_line":"            # fake EC reconstruction by returning first frag"},{"line_number":745,"context_line":"            return fragment_payload[0]"},{"line_number":746,"context_line":""},{"line_number":747,"context_line":"        reconstructor_logger \u003d FakeLogger()"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_0c48ae5a","line":744,"updated":"2016-10-13 08:50:41.000000000","message":"memo) this should be ok because each fragments should be same bytes length.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":760,"context_line":"                        \u0027device\u0027: self.device,"},{"line_number":761,"context_line":"                        \u0027partition\u0027: self.partition,"},{"line_number":762,"context_line":"                        \u0027policy\u0027: self.policy,"},{"line_number":763,"context_line":"                        # for a sync job we iterate over frag index that"},{"line_number":764,"context_line":"                        # belongs on local node"},{"line_number":765,"context_line":"                        \u0027frag_index\u0027: self.tx_node_index,"},{"line_number":766,"context_line":"                        \u0027sync_diskfile_builder\u0027:"},{"line_number":767,"context_line":"                            self.reconstructor.reconstruct_fa"},{"line_number":768,"context_line":"                    }"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_a7dc3be5","line":765,"range":{"start_line":763,"start_character":0,"end_line":765,"end_character":56},"updated":"2016-10-13 08:50:41.000000000","message":"nits) in sync job and if node index is available, frag_index doesn\u0027t seem to be referenced and when i tried to remove this, nothing failed. it\u0027s ok for safety to do like this though.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"20c049ce31061ec77737dbb7f5cccd20cbfa6949","unresolved":false,"context_lines":[{"line_number":760,"context_line":"                        \u0027device\u0027: self.device,"},{"line_number":761,"context_line":"                        \u0027partition\u0027: self.partition,"},{"line_number":762,"context_line":"                        \u0027policy\u0027: self.policy,"},{"line_number":763,"context_line":"                        # for a sync job we iterate over frag index that"},{"line_number":764,"context_line":"                        # belongs on local node"},{"line_number":765,"context_line":"                        \u0027frag_index\u0027: self.tx_node_index,"},{"line_number":766,"context_line":"                        \u0027sync_diskfile_builder\u0027:"},{"line_number":767,"context_line":"                            self.reconstructor.reconstruct_fa"},{"line_number":768,"context_line":"                    }"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_b456f627","line":765,"range":{"start_line":763,"start_character":0,"end_line":765,"end_character":56},"in_reply_to":"fa6399be_7bf0edfe","updated":"2016-10-14 06:40:42.000000000","message":"Done","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":760,"context_line":"                        \u0027device\u0027: self.device,"},{"line_number":761,"context_line":"                        \u0027partition\u0027: self.partition,"},{"line_number":762,"context_line":"                        \u0027policy\u0027: self.policy,"},{"line_number":763,"context_line":"                        # for a sync job we iterate over frag index that"},{"line_number":764,"context_line":"                        # belongs on local node"},{"line_number":765,"context_line":"                        \u0027frag_index\u0027: self.tx_node_index,"},{"line_number":766,"context_line":"                        \u0027sync_diskfile_builder\u0027:"},{"line_number":767,"context_line":"                            self.reconstructor.reconstruct_fa"},{"line_number":768,"context_line":"                    }"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_7bf0edfe","line":765,"range":{"start_line":763,"start_character":0,"end_line":765,"end_character":56},"in_reply_to":"fa6399be_a7dc3be5","updated":"2016-10-13 14:26:39.000000000","message":"yep [1] and self.job_node does have an index.\n\n[1] https://github.com/openstack/swift/blob/beb6e727a66e299ee6a96f1ed1df5cf9cf355de0/swift/obj/ssync_sender.py#L187-L187","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":772,"context_line":"                    sender()"},{"line_number":773,"context_line":"        return trace"},{"line_number":774,"context_line":""},{"line_number":775,"context_line":"    def test_sync_reconstructor_partial_rebuild(self):"},{"line_number":776,"context_line":"        frag_responses \u003d ["},{"line_number":777,"context_line":"            [FakeResponse(i, \u0027x\u0027 * (self.frag_length - 1))"},{"line_number":778,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)],"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_021a356e","line":775,"updated":"2016-10-13 08:50:41.000000000","message":"memo) so this test attempts that situation as follows:\n\n- try to sync x and y object via a sync\n- but x (the first object) is lack of the body\n- then, asserting nothing synced (including y) and log the error.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":772,"context_line":"                    sender()"},{"line_number":773,"context_line":"        return trace"},{"line_number":774,"context_line":""},{"line_number":775,"context_line":"    def test_sync_reconstructor_partial_rebuild(self):"},{"line_number":776,"context_line":"        frag_responses \u003d ["},{"line_number":777,"context_line":"            [FakeResponse(i, \u0027x\u0027 * (self.frag_length - 1))"},{"line_number":778,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)],"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_5bd2495d","line":775,"in_reply_to":"fa6399be_021a356e","updated":"2016-10-13 14:26:39.000000000","message":"right, maybe I should add that comment!","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":825,"context_line":"    def test_sync_reconstructor_no_responses(self):"},{"line_number":826,"context_line":"        frag_responses \u003d ["},{"line_number":827,"context_line":"            [None"},{"line_number":828,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)],"},{"line_number":829,"context_line":"            [FakeResponse(i, \u0027y\u0027 * self.frag_length)"},{"line_number":830,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)]]"},{"line_number":831,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_9dcf8a7e","line":828,"updated":"2016-10-13 08:50:41.000000000","message":"Ok, this is that you added inline comment in ssync sender. this will cause DiskFileError before \"send_put\" and then, it will be skipped successfully.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":825,"context_line":"    def test_sync_reconstructor_no_responses(self):"},{"line_number":826,"context_line":"        frag_responses \u003d ["},{"line_number":827,"context_line":"            [None"},{"line_number":828,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)],"},{"line_number":829,"context_line":"            [FakeResponse(i, \u0027y\u0027 * self.frag_length)"},{"line_number":830,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)]]"},{"line_number":831,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_3bbaf50a","line":828,"in_reply_to":"fa6399be_9dcf8a7e","updated":"2016-10-13 14:26:39.000000000","message":"correct","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":836,"context_line":"        self.assertEqual(1, len(results[\u0027tx_updates\u0027]))"},{"line_number":837,"context_line":"        self.assertFalse(results[\u0027rx_updates\u0027])"},{"line_number":838,"context_line":"        self.assertEqual(\u0027PUT\u0027, results[\u0027tx_updates\u0027][0].get(\u0027method\u0027))"},{"line_number":839,"context_line":"        synced_obj_name \u003d results[\u0027tx_updates\u0027][0].get(\u0027path\u0027)[-2:]"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        msgs \u003d []"},{"line_number":842,"context_line":"        obj_name \u003d synced_obj_name"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_825e4574","line":839,"updated":"2016-10-13 08:50:41.000000000","message":"nits) it could be worth to add a sanity:\n\n self.assertEqual(\u0027o2\u0027, synced_obj_name)","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"20c049ce31061ec77737dbb7f5cccd20cbfa6949","unresolved":false,"context_lines":[{"line_number":836,"context_line":"        self.assertEqual(1, len(results[\u0027tx_updates\u0027]))"},{"line_number":837,"context_line":"        self.assertFalse(results[\u0027rx_updates\u0027])"},{"line_number":838,"context_line":"        self.assertEqual(\u0027PUT\u0027, results[\u0027tx_updates\u0027][0].get(\u0027method\u0027))"},{"line_number":839,"context_line":"        synced_obj_name \u003d results[\u0027tx_updates\u0027][0].get(\u0027path\u0027)[-2:]"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        msgs \u003d []"},{"line_number":842,"context_line":"        obj_name \u003d synced_obj_name"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_97dc9474","line":839,"in_reply_to":"fa6399be_2d7d53b1","updated":"2016-10-14 06:40:42.000000000","message":"Make sense. And the hash value depends on the /etc/swift.conf prefix/suffix. that\u0027s why the assertion passed in my local but didn\u0027t at the gerrit gate. Done.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"81c587c26cb9405a48b4ed714acd274be42fd2a0","unresolved":false,"context_lines":[{"line_number":836,"context_line":"        self.assertEqual(1, len(results[\u0027tx_updates\u0027]))"},{"line_number":837,"context_line":"        self.assertFalse(results[\u0027rx_updates\u0027])"},{"line_number":838,"context_line":"        self.assertEqual(\u0027PUT\u0027, results[\u0027tx_updates\u0027][0].get(\u0027method\u0027))"},{"line_number":839,"context_line":"        synced_obj_name \u003d results[\u0027tx_updates\u0027][0].get(\u0027path\u0027)[-2:]"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        msgs \u003d []"},{"line_number":842,"context_line":"        obj_name \u003d synced_obj_name"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_2d7d53b1","line":839,"in_reply_to":"fa6399be_825e4574","updated":"2016-10-13 16:06:48.000000000","message":"I don\u0027t think the first fragment to be sync\u0027d is guaranteed to be \u0027o2\u0027 - it depends on the order the suffix set is iterated over in yield_hashes. That is why I learn the name of the first fragment here.","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2c02d18d69f9ed05abc1b1a8b4c80af56b579a66","unresolved":false,"context_lines":[{"line_number":866,"context_line":"        self.assertIn(\u0027Unable to get enough responses\u0027, log_lines[0])"},{"line_number":867,"context_line":""},{"line_number":868,"context_line":"    def test_sync_reconstructor_rebuild_ok(self):"},{"line_number":869,"context_line":"        # Sanity test for this class of tests. Both fragments geta full"},{"line_number":870,"context_line":"        # complement of responses and rebuild correctly."},{"line_number":871,"context_line":"        frag_responses \u003d ["},{"line_number":872,"context_line":"            [FakeResponse(i, \u0027x\u0027 * self.frag_length)"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_1b5451c8","line":869,"updated":"2016-10-13 14:26:39.000000000","message":"nit: s/geta/get/","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"fd9cbc96badcfa6fe6dce5166a6fdce0675c08bf","unresolved":false,"context_lines":[{"line_number":893,"context_line":"            self.fail(\u0027Failed with:\\n%s\u0027 % \u0027\\n\u0027.join(msgs))"},{"line_number":894,"context_line":"        self.assertIn(\u0027x\u0027 * self.frag_length, rx_frags)"},{"line_number":895,"context_line":"        self.assertIn(\u0027y\u0027 * self.frag_length, rx_frags)"},{"line_number":896,"context_line":"        self.assertFalse(rx_frags[2:])"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":""},{"line_number":899,"context_line":"@patch_policies"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa6399be_7d39d640","line":896,"updated":"2016-10-13 08:50:41.000000000","message":"Yey, it looks healthy!","commit_id":"beb6e727a66e299ee6a96f1ed1df5cf9cf355de0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dacb62d6d2f354d95789e3c5716499b85d93a989","unresolved":false,"context_lines":[{"line_number":797,"context_line":"                      log_lines[0])"},{"line_number":798,"context_line":"        self.assertFalse(log_lines[1:])"},{"line_number":799,"context_line":"        # trampoline for the receiver to write a log"},{"line_number":800,"context_line":"        eventlet.sleep(0)"},{"line_number":801,"context_line":"        log_lines \u003d self.rx_logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":802,"context_line":"        self.assertIn(\u0027ssync subrequest failed with 499\u0027,"},{"line_number":803,"context_line":"                      log_lines[0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_14331763","line":800,"updated":"2016-10-14 13:01:52.000000000","message":"when I backported this patch to origin/stable/mitaka (no conflcits) this test failed at line 802 because the rx_loggger had not yet been written. It passes if I add \u003e0 sleep or sleep(1) twice. I don\u0027t yet understand why - the trampoline yields to the receiver, or is there another coro and the receiver doesn\u0027t get to run even with this trampoline??\n\nSo I am concerned this may just happen to pass on master but is flakey. That said, I have cycled this test on two different VMs on master branch and seen zero failures.","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"32933da7c5a242207f99ad1699f9ff6edf877c36","unresolved":false,"context_lines":[{"line_number":797,"context_line":"                      log_lines[0])"},{"line_number":798,"context_line":"        self.assertFalse(log_lines[1:])"},{"line_number":799,"context_line":"        # trampoline for the receiver to write a log"},{"line_number":800,"context_line":"        eventlet.sleep(0)"},{"line_number":801,"context_line":"        log_lines \u003d self.rx_logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":802,"context_line":"        self.assertIn(\u0027ssync subrequest failed with 499\u0027,"},{"line_number":803,"context_line":"                      log_lines[0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_ef4313ad","line":800,"in_reply_to":"fa6399be_14331763","updated":"2016-10-14 22:33:56.000000000","message":"IMHO I think your gut is right, I\u0027ve come to trust these sorts of \"just trampoline x times\" fixes less and less; and favor instead standard concurrency test robustness:\n\ni.e. \n\n    while \u003c timeout:\n       if thing checks out:\n           break\n    else:\n        self.fail(\u0027gave up after timeout\u0027)\n\n\nBut this WOMM as written and would advocate for now we leave it as written for clarity.  I think the above should only be used if it gives us problems (i.e. in backports).","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c3a7b71defec7452523485adc21e2b0a9aa18976","unresolved":false,"context_lines":[{"line_number":849,"context_line":"             for i in range(self.policy.ec_ndata)] +"},{"line_number":850,"context_line":"            # ec_nparity responses will raise an Exception - at least one of"},{"line_number":851,"context_line":"            # these will be used during rebuild"},{"line_number":852,"context_line":"            [FakeResponse(i, Exception(\u0027raised in response read method\u0027))"},{"line_number":853,"context_line":"             for i in range(self.policy.ec_ndata,"},{"line_number":854,"context_line":"                            self.policy.ec_ndata + self.policy.ec_nparity)],"},{"line_number":855,"context_line":"            # second set of response are all good"},{"line_number":856,"context_line":"            [FakeResponse(i, \u0027y\u0027 * self.frag_length)"},{"line_number":857,"context_line":"             for i in range(self.policy.ec_ndata + self.policy.ec_nparity)]]"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa6399be_97401436","line":854,"range":{"start_line":852,"start_character":13,"end_line":854,"end_character":76},"updated":"2016-10-14 06:44:19.000000000","message":"It\u0027s ok for now though, this causes the exception stacktrace to STDERR when running `nosetests test/unit/obj/test_ssync.py` that is because the exception raised in the eventlet hub. Exactly, the eventlet exception will be caught at [1] so in the swift perspective, it\u0027s safe.\n\nWill try to improve the error handling around eventlet hub but I\u0027d like to make it in another follow up because the improvement may be taken more time to discuss.\n\n1: https://github.com/openstack/swift/blob/master/swift/obj/reconstructor.py#L326","commit_id":"3218f8b064e462d901466b04a4813e15ec96da85"}]}