)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-11-05 14:08:51 -0600"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Debug EC multipart/byteranges responses"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"We\u0027re seeing some EC responses hit ChunkReadTimeout and fail to resume."},{"line_number":10,"context_line":"There\u0027s a bug in here and I\u0027m not sure where yet."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1f621f24_01d9df7d","line":7,"updated":"2020-11-09 19:04:31.000000000","message":"maybe highlight the substantive changes in the subject e.g. change to the exception raised","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Debug EC multipart/byteranges responses"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"We\u0027re seeing some EC responses hit ChunkReadTimeout and fail to resume."},{"line_number":10,"context_line":"There\u0027s a bug in here and I\u0027m not sure where yet."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Change-Id: I0654815543be3df059eb2875d9b3669dbd97f5b4"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1f621f24_81db2f7a","line":9,"updated":"2020-11-09 19:04:31.000000000","message":"do we want this assertion in the git log...forever?","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0a4a7e46773d7ce1be5fb736e904b2b3e152a543","unresolved":false,"context_lines":[{"line_number":2530,"context_line":"                                new_source,"},{"line_number":2531,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2532,"context_line":"                        else:"},{"line_number":2533,"context_line":"                            raise StopIteration()"},{"line_number":2534,"context_line":""},{"line_number":2535,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2536,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_41366554","side":"PARENT","line":2533,"updated":"2020-11-05 00:19:43.000000000","message":"Yeah, this *does* seem suspicious...","commit_id":"d825fd66bd8f4cfc559667f6fedf4e6e47f72d1b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a1e9d3e4b685cdd49c2a292e0b1591e56ecf1b80","unresolved":false,"context_lines":[{"line_number":2530,"context_line":"                                new_source,"},{"line_number":2531,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2532,"context_line":"                        else:"},{"line_number":2533,"context_line":"                            raise StopIteration()"},{"line_number":2534,"context_line":""},{"line_number":2535,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2536,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_c1c6f548","side":"PARENT","line":2533,"updated":"2020-11-05 00:08:51.000000000","message":"this is super sketch - we\u0027re converting a ChunkReadTimeout to a StopIteration because we couldn\u0027t find another node.  That makes it look like we just finished handling all the ranges?!","commit_id":"d825fd66bd8f4cfc559667f6fedf4e6e47f72d1b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a1e9d3e4b685cdd49c2a292e0b1591e56ecf1b80","unresolved":false,"context_lines":[{"line_number":2664,"context_line":"                           \u0027part_iter\u0027: part_iter}"},{"line_number":2665,"context_line":"                    self.pop_range()"},{"line_number":2666,"context_line":"            except StopIteration:"},{"line_number":2667,"context_line":"                req.environ[\u0027swift.non_client_disconnect\u0027] \u003d True"},{"line_number":2668,"context_line":"            finally:"},{"line_number":2669,"context_line":"                if part_iter:"},{"line_number":2670,"context_line":"                    part_iter.close()"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_01de2d10","side":"PARENT","line":2667,"updated":"2020-11-05 00:08:51.000000000","message":"I think this was just too wide, and it might have been doubling up the ChunkReadTimeout handling from the error conversion above.","commit_id":"d825fd66bd8f4cfc559667f6fedf4e6e47f72d1b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a1e9d3e4b685cdd49c2a292e0b1591e56ecf1b80","unresolved":false,"context_lines":[{"line_number":2552,"context_line":"                        try:"},{"line_number":2553,"context_line":"                            self.fast_forward(self.bytes_used_from_backend)"},{"line_number":2554,"context_line":"                        except (HTTPException, ValueError):"},{"line_number":2555,"context_line":"                            self.app.logger.exception(\u0027Unable to fast forward\u0027)"},{"line_number":2556,"context_line":"                            six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2557,"context_line":"                        except RangeAlreadyComplete:"},{"line_number":2558,"context_line":"                            break"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_2104097b","line":2555,"updated":"2020-11-05 00:08:51.000000000","message":"this is the traceback I\u0027m most interested in\n\ni\u0027m still trying to track down the same thing I was hunting with https://review.opendev.org/#/c/754457/","commit_id":"7a64e208871f12dc2c5efc0c1b419481483fe71f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a1e9d3e4b685cdd49c2a292e0b1591e56ecf1b80","unresolved":false,"context_lines":[{"line_number":2583,"context_line":"                                six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2584,"context_line":"                            part_file \u003d ByteCountEnforcer(part_file, nbytes)"},{"line_number":2585,"context_line":"                        else:"},{"line_number":2586,"context_line":"                            six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2587,"context_line":"                    else:"},{"line_number":2588,"context_line":"                        if buf and self.skip_bytes:"},{"line_number":2589,"context_line":"                            if self.skip_bytes \u003c len(buf):"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_41e825b3","line":2586,"updated":"2020-11-05 00:08:51.000000000","message":"One or both of these re-raises\u0027 aren\u0027t getting tested","commit_id":"7a64e208871f12dc2c5efc0c1b419481483fe71f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0a4a7e46773d7ce1be5fb736e904b2b3e152a543","unresolved":false,"context_lines":[{"line_number":2803,"context_line":"                    \u0027X-Object-Sysmeta-EC-ETag\u0027) \u003d\u003d used_etag:"},{"line_number":2804,"context_line":"                return source, node"},{"line_number":2805,"context_line":"            else:"},{"line_number":2806,"context_line":"                self.app.logger.warning(\u0027Skipping source %s\u0027 % source.status)"},{"line_number":2807,"context_line":"        return None, None"},{"line_number":2808,"context_line":""},{"line_number":2809,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_e117b1a2","line":2806,"updated":"2020-11-05 00:19:43.000000000","message":"Might want to make this something more like\n\n elif not is_good_source(source.status):\n     self.app.logger.warning(\n         \u0027Skipping source (status %s)\u0027, source.status)\n else:\n     self.app.logger.warning(\n         \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,\n         source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027), used_etag)","commit_id":"7a64e208871f12dc2c5efc0c1b419481483fe71f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2a5eacda5084a5c53a9af3b6ce009a009b9b39c0","unresolved":false,"context_lines":[{"line_number":2803,"context_line":"                    \u0027X-Object-Sysmeta-EC-ETag\u0027) \u003d\u003d used_etag:"},{"line_number":2804,"context_line":"                return source, node"},{"line_number":2805,"context_line":"            else:"},{"line_number":2806,"context_line":"                self.app.logger.warning(\u0027Skipping source %s\u0027 % source.status)"},{"line_number":2807,"context_line":"        return None, None"},{"line_number":2808,"context_line":""},{"line_number":2809,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_3ac70248","line":2806,"in_reply_to":"1f621f24_e117b1a2","updated":"2020-11-05 20:12:33.000000000","message":"Done","commit_id":"7a64e208871f12dc2c5efc0c1b419481483fe71f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ed440233b4502b54574a80c39827e7a4de029cd9","unresolved":false,"context_lines":[{"line_number":1402,"context_line":"                    queue.put(fragment)"},{"line_number":1403,"context_line":"            except GreenletExit:"},{"line_number":1404,"context_line":"                # killed by contextpool"},{"line_number":1405,"context_line":"                pass"},{"line_number":1406,"context_line":"            except ChunkReadTimeout:"},{"line_number":1407,"context_line":"                # unable to resume in ECFragGetter"},{"line_number":1408,"context_line":"                self.logger.exception(_(\"Timeout fetching fragments for %r\"),"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_eabf5d5e","line":1405,"updated":"2020-11-09 17:33:55.000000000","message":"do we need or want a message on this one too?","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":2509,"context_line":"                            # raises StopIteration)."},{"line_number":2510,"context_line":"                            # Otherwise, this call to next() performs IO when"},{"line_number":2511,"context_line":"                            # we have a multipart/byteranges response; as it"},{"line_number":2512,"context_line":"                            # will read the MIME boundary and part headers."},{"line_number":2513,"context_line":"                            start_byte, end_byte, length, headers, part \u003d next("},{"line_number":2514,"context_line":"                                parts_iter[0])"},{"line_number":2515,"context_line":"                        return (start_byte, end_byte, length, headers, part)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_befbeb44","line":2512,"updated":"2020-11-09 19:04:31.000000000","message":"+1 clearer commenting","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2a5eacda5084a5c53a9af3b6ce009a009b9b39c0","unresolved":false,"context_lines":[{"line_number":2530,"context_line":"                                new_source,"},{"line_number":2531,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2532,"context_line":"                        else:"},{"line_number":2533,"context_line":"                            raise"},{"line_number":2534,"context_line":""},{"line_number":2535,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2536,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_fd81044c","line":2533,"updated":"2020-11-05 20:12:33.000000000","message":"for all the new tests; there\u0027s really only one that fails slightly differently (worse?) without this change\n\nMaybe it was sketchy; and maybe this is better - but I don\u0027t think it\u0027s \"the bug\" I\u0027m trying to chase down.","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":2530,"context_line":"                                new_source,"},{"line_number":2531,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2532,"context_line":"                        else:"},{"line_number":2533,"context_line":"                            raise"},{"line_number":2534,"context_line":""},{"line_number":2535,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2536,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_2920479b","line":2533,"in_reply_to":"1f621f24_fd81044c","updated":"2020-11-09 19:04:31.000000000","message":"this is now consistent with line 2586 (except six.reraise is used there)","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":2579,"context_line":"                            except StopIteration:"},{"line_number":2580,"context_line":"                                # it\u0027s not clear to me how to make"},{"line_number":2581,"context_line":"                                # get_next_doc_part raise StopIteration for the"},{"line_number":2582,"context_line":"                                # first doc part of a new request"},{"line_number":2583,"context_line":"                                six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2584,"context_line":"                            part_file \u003d ByteCountEnforcer(part_file, nbytes)"},{"line_number":2585,"context_line":"                        else:"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_c9cd5332","line":2582,"updated":"2020-11-09 19:04:31.000000000","message":"agree, given change at line 2533","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ed440233b4502b54574a80c39827e7a4de029cd9","unresolved":false,"context_lines":[{"line_number":2682,"context_line":"                _(\u0027Client did not read from proxy within %ss\u0027) %"},{"line_number":2683,"context_line":"                self.app.client_timeout)"},{"line_number":2684,"context_line":"            self.app.logger.increment(\u0027client_timeouts\u0027)"},{"line_number":2685,"context_line":"        except GeneratorExit:"},{"line_number":2686,"context_line":"            warn \u003d True"},{"line_number":2687,"context_line":"            req_range \u003d self.backend_headers[\u0027Range\u0027]"},{"line_number":2688,"context_line":"            if req_range:"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_6a786dc3","line":2685,"updated":"2020-11-09 17:33:55.000000000","message":"there might need to be a log line here too!","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":2803,"context_line":"                continue"},{"line_number":2804,"context_line":"            if not is_good_source(source.status):"},{"line_number":2805,"context_line":"                self.app.logger.warning("},{"line_number":2806,"context_line":"                    \u0027Skipping source (status %s)\u0027, source.status)"},{"line_number":2807,"context_line":"            elif source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027) !\u003d used_etag:"},{"line_number":2808,"context_line":"                self.app.logger.warning("},{"line_number":2809,"context_line":"                    \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_eabd9dd8","line":2806,"updated":"2020-11-09 19:04:31.000000000","message":"I don\u0027t *think* bad sources make it out of make_node_request in the source and node iter, so not sure this will ever be reached","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1849d8d14595aba76cc779a6f1454153a4d166c2","unresolved":false,"context_lines":[{"line_number":2803,"context_line":"                continue"},{"line_number":2804,"context_line":"            if not is_good_source(source.status):"},{"line_number":2805,"context_line":"                self.app.logger.warning("},{"line_number":2806,"context_line":"                    \u0027Skipping source (status %s)\u0027, source.status)"},{"line_number":2807,"context_line":"            elif source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027) !\u003d used_etag:"},{"line_number":2808,"context_line":"                self.app.logger.warning("},{"line_number":2809,"context_line":"                    \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_4a6bf11c","line":2806,"updated":"2020-11-09 17:20:18.000000000","message":"this may never pop because the source iter may already filter out bad responses and keep going or return None (maybe?)","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":2807,"context_line":"            elif source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027) !\u003d used_etag:"},{"line_number":2808,"context_line":"                self.app.logger.warning("},{"line_number":2809,"context_line":"                    \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,"},{"line_number":2810,"context_line":"                    source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027), used_etag)"},{"line_number":2811,"context_line":"            else:"},{"line_number":2812,"context_line":"                return source, node"},{"line_number":2813,"context_line":"        return None, None"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_cac26159","line":2810,"updated":"2020-11-09 19:04:31.000000000","message":"useful addition","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"11290d38a1740bd45a392dd1806855500276c50c","unresolved":false,"context_lines":[{"line_number":1409,"context_line":"                                      quote(self.path))"},{"line_number":1410,"context_line":"            except:  # noqa"},{"line_number":1411,"context_line":"                self.logger.exception(\"Exception fetching fragments for %r\","},{"line_number":1412,"context_line":"                                      quote(self.path))"},{"line_number":1413,"context_line":"            finally:"},{"line_number":1414,"context_line":"                queue.resize(2)  # ensure there\u0027s room"},{"line_number":1415,"context_line":"                queue.put(None)"}],"source_content_type":"text/x-python","patch_set":3,"id":"fffc6b78_669fa8b4","line":1412,"updated":"2020-11-19 20:48:20.000000000","message":"I have a bunch of ChunkWriteTimeout leaking through here - which I guess means the client was too slow to read - there\u0027s no resume attempts\n\nWe are \"fixing\" the status for access logging - on a ranged EC GET i\u0027m seeing 500 - when it\u0027s an SLO segment we\u0027re logging 499.","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b5edf463ad1e990e2270e24ac0b6421abe6b31fc","unresolved":false,"context_lines":[{"line_number":2506,"context_line":"                            # If we don\u0027t have a multipart/byteranges response,"},{"line_number":2507,"context_line":"                            # but just a 200 or a single-range 206, then this"},{"line_number":2508,"context_line":"                            # performs no IO, and just returns source (or"},{"line_number":2509,"context_line":"                            # raises StopIteration)."},{"line_number":2510,"context_line":"                            # Otherwise, this call to next() performs IO when"},{"line_number":2511,"context_line":"                            # we have a multipart/byteranges response; as it"},{"line_number":2512,"context_line":"                            # will read the MIME boundary and part headers."}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_5434054a","line":2509,"updated":"2020-11-12 17:55:05.000000000","message":"raises StopIteration if we\u0027d already let out the one-and-only part and it gets called again","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"11290d38a1740bd45a392dd1806855500276c50c","unresolved":false,"context_lines":[{"line_number":2552,"context_line":"                        try:"},{"line_number":2553,"context_line":"                            self.fast_forward(self.bytes_used_from_backend)"},{"line_number":2554,"context_line":"                        except (HTTPException, ValueError):"},{"line_number":2555,"context_line":"                            self.app.logger.exception(\u0027Unable to fast forward\u0027)"},{"line_number":2556,"context_line":"                            six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2557,"context_line":"                        except RangeAlreadyComplete:"},{"line_number":2558,"context_line":"                            break"}],"source_content_type":"text/x-python","patch_set":3,"id":"fffc6b78_e63c38c1","line":2555,"updated":"2020-11-19 20:48:20.000000000","message":"sadly, none of these","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0e451c79f2e54818df11241cf17f227aa5d130c6","unresolved":false,"context_lines":[{"line_number":2561,"context_line":"                        if new_source:"},{"line_number":2562,"context_line":"                            self.app.error_occurred("},{"line_number":2563,"context_line":"                                self.node, _(\u0027Trying to read object during \u0027"},{"line_number":2564,"context_line":"                                             \u0027GET (retrying)\u0027))"},{"line_number":2565,"context_line":"                            # Close-out the connection as best as possible."},{"line_number":2566,"context_line":"                            if getattr(self.source, \u0027swift_conn\u0027, None):"},{"line_number":2567,"context_line":"                                close_swift_conn(self.source)"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_f7c393eb","line":2564,"updated":"2020-11-12 19:17:34.000000000","message":"We should maybe update this message (and the one up at L2521) to clarify that we\u0027re in the EC get path (instead of the one up in base.py) just so we know we\u0027re hitting this code path *at all*.","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4cc227903608553a833067872cc150409bb1fb30","unresolved":false,"context_lines":[{"line_number":2650,"context_line":"                            get_next_doc_part()"},{"line_number":2651,"context_line":"                    except StopIteration:"},{"line_number":2652,"context_line":"                        # it seems this is the only way out of the loop; not"},{"line_number":2653,"context_line":"                        # sure why the req.environ update is always needed"},{"line_number":2654,"context_line":"                        req.environ[\u0027swift.non_client_disconnect\u0027] \u003d True"},{"line_number":2655,"context_line":"                        break"},{"line_number":2656,"context_line":"                    # note: learn_size_from_content_range() sets"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_e0ee3020","line":2653,"updated":"2020-11-17 18:52:44.000000000","message":"right, this seems to be the *expected* termination of reading doc parts","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b5edf463ad1e990e2270e24ac0b6421abe6b31fc","unresolved":false,"context_lines":[{"line_number":2664,"context_line":"                                  if (end_byte is not None"},{"line_number":2665,"context_line":"                                      and start_byte is not None)"},{"line_number":2666,"context_line":"                                  else None)"},{"line_number":2667,"context_line":"                    part_iter \u003d iter_bytes_from_response_part(part, byte_count)"},{"line_number":2668,"context_line":"                    yield {\u0027start_byte\u0027: start_byte, \u0027end_byte\u0027: end_byte,"},{"line_number":2669,"context_line":"                           \u0027entity_length\u0027: length, \u0027headers\u0027: headers,"},{"line_number":2670,"context_line":"                           \u0027part_iter\u0027: part_iter}"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_b4d479ea","line":2667,"updated":"2020-11-12 17:55:05.000000000","message":"do we ever raise StopIteration from *here*?!","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4cc227903608553a833067872cc150409bb1fb30","unresolved":false,"context_lines":[{"line_number":2668,"context_line":"                    yield {\u0027start_byte\u0027: start_byte, \u0027end_byte\u0027: end_byte,"},{"line_number":2669,"context_line":"                           \u0027entity_length\u0027: length, \u0027headers\u0027: headers,"},{"line_number":2670,"context_line":"                           \u0027part_iter\u0027: part_iter}"},{"line_number":2671,"context_line":"                    self.pop_range()"},{"line_number":2672,"context_line":"            finally:"},{"line_number":2673,"context_line":"                if part_iter:"},{"line_number":2674,"context_line":"                    part_iter.close()"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_20f5c8ed","line":2671,"updated":"2020-11-17 18:52:44.000000000","message":"the StopIteration that may have previously been raised by a read timeout in get_next_doc_part has now been morphed to a ChunkReadTimeout, which is caught below, but we\u0027ve lost the \n\u0027swift.non_client_disconnect\u0027 \u003d True for that case.\n\nThat said we don\u0027t \u0027swift.non_client_disconnect\u0027 \u003d True for other read timeouts e.g. in the bytes iter","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b5edf463ad1e990e2270e24ac0b6421abe6b31fc","unresolved":false,"context_lines":[{"line_number":2671,"context_line":"                    self.pop_range()"},{"line_number":2672,"context_line":"            finally:"},{"line_number":2673,"context_line":"                if part_iter:"},{"line_number":2674,"context_line":"                    part_iter.close()"},{"line_number":2675,"context_line":""},{"line_number":2676,"context_line":"        except ChunkReadTimeout:"},{"line_number":2677,"context_line":"            self.app.exception_occurred(self.node, \u0027Object\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f621f24_54e6c5e9","line":2674,"updated":"2020-11-12 17:55:05.000000000","message":"maybe let\u0027s move this down with the other finally","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"11290d38a1740bd45a392dd1806855500276c50c","unresolved":false,"context_lines":[{"line_number":2752,"context_line":"            # throw out 5XX and 404s from handoff nodes unless the data is"},{"line_number":2753,"context_line":"            # really on disk and had been DELETEd"},{"line_number":2754,"context_line":"            self.app.logger.info("},{"line_number":2755,"context_line":"                \u0027Ignoring %s from handoff\u0027 % possible_source.status)"},{"line_number":2756,"context_line":"            conn.close()"},{"line_number":2757,"context_line":"            return None"},{"line_number":2758,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"fffc6b78_8330be09","line":2755,"updated":"2020-11-19 20:48:20.000000000","message":"when this is just a 404 it\u0027s pretty chatty","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"11290d38a1740bd45a392dd1806855500276c50c","unresolved":false,"context_lines":[{"line_number":2776,"context_line":"                     \u0027body\u0027: self.body[:1024]})"},{"line_number":2777,"context_line":"            else:"},{"line_number":2778,"context_line":"                self.app.logger.info("},{"line_number":2779,"context_line":"                    \u0027Ignoring %s from primary\u0027 % possible_source.status)"},{"line_number":2780,"context_line":""},{"line_number":2781,"context_line":"            return None"},{"line_number":2782,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"fffc6b78_e63718e3","line":2779,"updated":"2020-11-19 20:48:20.000000000","message":"when this is just 404 it\u0027s pretty chatty","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"11290d38a1740bd45a392dd1806855500276c50c","unresolved":false,"context_lines":[{"line_number":2809,"context_line":"            if source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027) !\u003d used_etag:"},{"line_number":2810,"context_line":"                self.app.logger.warning("},{"line_number":2811,"context_line":"                    \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,"},{"line_number":2812,"context_line":"                    source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027), used_etag)"},{"line_number":2813,"context_line":"            else:"},{"line_number":2814,"context_line":"                return source, node"},{"line_number":2815,"context_line":"        return None, None"}],"source_content_type":"text/x-python","patch_set":3,"id":"fffc6b78_c6565407","line":2812,"updated":"2020-11-19 20:48:20.000000000","message":"none of these","commit_id":"7ab39624ac8502fc20869dc853b60852a98a6ed2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2dee6874c68809c3815cce6abc37c6e6bd149859","unresolved":true,"context_lines":[{"line_number":2552,"context_line":"                        try:"},{"line_number":2553,"context_line":"                            self.fast_forward(self.bytes_used_from_backend)"},{"line_number":2554,"context_line":"                        except (HTTPException, ValueError):"},{"line_number":2555,"context_line":"                            self.app.logger.exception(\u0027Unable to fast forward\u0027)"},{"line_number":2556,"context_line":"                            six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":2557,"context_line":"                        except RangeAlreadyComplete:"},{"line_number":2558,"context_line":"                            break"}],"source_content_type":"text/x-python","patch_set":5,"id":"b035cedf_1e56acf0","line":2555,"updated":"2021-01-08 21:21:07.000000000","message":"apparently this can be a 416:\n\n\t2021-01-02T17:40:11.782128+00:00 ss0118 proxy-server: err Unable to fast forward: \n\tTraceback (most recent call last):\n\t  File \"/opt/ss/lib/python2.7/site-packages/swift/proxy/controllers/obj.py\", line 2553, in iter_bytes_from_response_part\n\t    self.fast_forward(self.bytes_used_from_backend)\n\t  File \"/opt/ss/lib/python2.7/site-packages/swift/proxy/controllers/obj.py\", line 2409, in fast_forward\n\t    raise HTTPRequestedRangeNotSatisfiable()\n\tHTTPException: 416 Requested Range Not Satisfiable","commit_id":"0864049bf6a7612613878860730d374dc49fce13"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"2dee6874c68809c3815cce6abc37c6e6bd149859","unresolved":true,"context_lines":[{"line_number":2809,"context_line":"            if source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027) !\u003d used_etag:"},{"line_number":2810,"context_line":"                self.app.logger.warning("},{"line_number":2811,"context_line":"                    \u0027Skipping source (etag mismatch: got %s, expected %s)\u0027,"},{"line_number":2812,"context_line":"                    source.getheader(\u0027X-Object-Sysmeta-EC-ETag\u0027), used_etag)"},{"line_number":2813,"context_line":"            else:"},{"line_number":2814,"context_line":"                return source, node"},{"line_number":2815,"context_line":"        return None, None"}],"source_content_type":"text/x-python","patch_set":5,"id":"1ba815c1_74123240","line":2812,"updated":"2021-01-08 21:21:07.000000000","message":"so, this happens:\n\n2021-01-03T01:30:45.512461+00:00 ss0964 proxy-server: warning Skipping source (etag mismatch: got 23caf7eee2d3551a286702dce576ac7b, expected e6c3050a7370619fb5172297f9f9ae1f)","commit_id":"0864049bf6a7612613878860730d374dc49fce13"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d24086a14e8786eb495e54ca46fb86065d495eaf","unresolved":true,"context_lines":[{"line_number":2517,"context_line":"                            # Otherwise, this call to next() performs IO when"},{"line_number":2518,"context_line":"                            # we have a multipart/byteranges response; as it"},{"line_number":2519,"context_line":"                            # will read the MIME boundary and part headers."},{"line_number":2520,"context_line":"                            start_byte, end_byte, length, headers, part \u003d next("},{"line_number":2521,"context_line":"                                parts_iter[0])"},{"line_number":2522,"context_line":"                        return (start_byte, end_byte, length, headers, part)"},{"line_number":2523,"context_line":"                    except ChunkReadTimeout:"}],"source_content_type":"text/x-python","patch_set":7,"id":"fd58074f_23a0e467","line":2520,"range":{"start_line":2520,"start_character":74,"end_line":2520,"end_character":78},"updated":"2021-04-21 18:34:48.000000000","message":"This is now the only path that should have get_next_doc_part() raise StopIteration, right?","commit_id":"4a4d899680736c85d2982e23e1f6becedab6ea7f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d4f3b6046f532ccc2311d185f2716b64257510eb","unresolved":true,"context_lines":[{"line_number":2537,"context_line":"                                new_source,"},{"line_number":2538,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2539,"context_line":"                        else:"},{"line_number":2540,"context_line":"                            raise"},{"line_number":2541,"context_line":""},{"line_number":2542,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2543,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":7,"id":"f1edcb89_c0f3f8a7","line":2540,"updated":"2021-04-21 18:08:15.000000000","message":"this change is functionally significant for multi-range requests; the existing code would truncate results instead of raising the error.\n\nswift/test/unit/proxy/controllers/test_obj.py:TestECObjController.test_GET_with_multirange_unable_to_resume","commit_id":"4a4d899680736c85d2982e23e1f6becedab6ea7f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d24086a14e8786eb495e54ca46fb86065d495eaf","unresolved":true,"context_lines":[{"line_number":2537,"context_line":"                                new_source,"},{"line_number":2538,"context_line":"                                read_chunk_size\u003dself.app.object_chunk_size)"},{"line_number":2539,"context_line":"                        else:"},{"line_number":2540,"context_line":"                            raise"},{"line_number":2541,"context_line":""},{"line_number":2542,"context_line":"            def iter_bytes_from_response_part(part_file, nbytes):"},{"line_number":2543,"context_line":"                nchunks \u003d 0"}],"source_content_type":"text/x-python","patch_set":7,"id":"99df8582_3ebd842b","line":2540,"in_reply_to":"f1edcb89_c0f3f8a7","updated":"2021-04-21 18:34:48.000000000","message":"I think it\u0027s also the *only* functional change now.","commit_id":"4a4d899680736c85d2982e23e1f6becedab6ea7f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d4f3b6046f532ccc2311d185f2716b64257510eb","unresolved":true,"context_lines":[{"line_number":2658,"context_line":"                        # it seems this is the only way out of the loop; not"},{"line_number":2659,"context_line":"                        # sure why the req.environ update is always needed"},{"line_number":2660,"context_line":"                        req.environ[\u0027swift.non_client_disconnect\u0027] \u003d True"},{"line_number":2661,"context_line":"                        break"},{"line_number":2662,"context_line":"                    # note: learn_size_from_content_range() sets"},{"line_number":2663,"context_line":"                    # self.skip_bytes"},{"line_number":2664,"context_line":"                    self.learn_size_from_content_range("}],"source_content_type":"text/x-python","patch_set":7,"id":"48a8a10c_5d53bb91","line":2661,"updated":"2021-04-21 18:08:15.000000000","message":"as best I can tell from tests, this change is *entirely* functionally equivilant.  i.e. the only code that might raise StopIteration was get_next_doc_part and handling it here this way is effectively the same, but tighter and more obvious what\u0027s going on where.","commit_id":"4a4d899680736c85d2982e23e1f6becedab6ea7f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d24086a14e8786eb495e54ca46fb86065d495eaf","unresolved":true,"context_lines":[{"line_number":2658,"context_line":"                        # it seems this is the only way out of the loop; not"},{"line_number":2659,"context_line":"                        # sure why the req.environ update is always needed"},{"line_number":2660,"context_line":"                        req.environ[\u0027swift.non_client_disconnect\u0027] \u003d True"},{"line_number":2661,"context_line":"                        break"},{"line_number":2662,"context_line":"                    # note: learn_size_from_content_range() sets"},{"line_number":2663,"context_line":"                    # self.skip_bytes"},{"line_number":2664,"context_line":"                    self.learn_size_from_content_range("}],"source_content_type":"text/x-python","patch_set":7,"id":"c5d908f6_8613fc16","line":2661,"in_reply_to":"48a8a10c_5d53bb91","updated":"2021-04-21 18:34:48.000000000","message":"Yeah, I was a little nervous about what could happen with the `yield` -- but I think the only way to have that raise would be something like\n\n app_iter.throw(StopIteration)\n\nwhich is *definitely* not the sort of idiom we\u0027d use.","commit_id":"4a4d899680736c85d2982e23e1f6becedab6ea7f"}],"test/unit/__init__.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":1194,"context_line":"class StubResponse(object):"},{"line_number":1195,"context_line":""},{"line_number":1196,"context_line":"    def __init__(self, status, body\u003db\u0027\u0027, headers\u003dNone, frag_index\u003dNone,"},{"line_number":1197,"context_line":"                 slowdown\u003dNone):"},{"line_number":1198,"context_line":"        self.status \u003d status"},{"line_number":1199,"context_line":"        self.body \u003d body"},{"line_number":1200,"context_line":"        self.readable \u003d BytesIO(body)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_6ffa3fbd","line":1197,"updated":"2020-11-09 19:04:31.000000000","message":"just curious, I haven\u0027t fully reloaded the test infrastructure into my head...does SlowBody not do a similar thing to adding slowdown here?","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":4005,"context_line":""},{"line_number":4006,"context_line":"    def test_GET_with_multirange(self):"},{"line_number":4007,"context_line":"        self.app.object_chunk_size \u003d 256"},{"line_number":4008,"context_line":"        ec_stub \u003d self._make_ec_object_stub()"},{"line_number":4009,"context_line":"        test_body \u003d b\u0027test\u0027 * self.policy.ec_segment_size"},{"line_number":4010,"context_line":"        ec_stub \u003d make_ec_object_stub(test_body, self.policy, None)"},{"line_number":4011,"context_line":"        frag_archives \u003d ec_stub[\u0027frags\u0027]"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_2f7cc770","line":4008,"updated":"2020-11-09 19:04:31.000000000","message":"unused/reassigned line 4010","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":4075,"context_line":"    def test_GET_with_multirange_slow_body(self):"},{"line_number":4076,"context_line":"        self.app.object_chunk_size \u003d 256"},{"line_number":4077,"context_line":"        self.app.recoverable_node_timeout \u003d 0.01"},{"line_number":4078,"context_line":"        ec_stub \u003d self._make_ec_object_stub()"},{"line_number":4079,"context_line":"        test_body \u003d b\u0027test\u0027 * self.policy.ec_segment_size"},{"line_number":4080,"context_line":"        ec_stub \u003d make_ec_object_stub(test_body, self.policy, None)"},{"line_number":4081,"context_line":"        frag_archives \u003d ec_stub[\u0027frags\u0027]"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_0f810b4b","line":4078,"updated":"2020-11-09 19:04:31.000000000","message":"unused - reassigned line 4080","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":4147,"context_line":"    def test_GET_with_multirange_unable_to_resume(self):"},{"line_number":4148,"context_line":"        self.app.object_chunk_size \u003d 256"},{"line_number":4149,"context_line":"        self.app.recoverable_node_timeout \u003d 0.01"},{"line_number":4150,"context_line":"        ec_stub \u003d self._make_ec_object_stub()"},{"line_number":4151,"context_line":"        test_body \u003d b\u0027test\u0027 * self.policy.ec_segment_size"},{"line_number":4152,"context_line":"        ec_stub \u003d make_ec_object_stub(test_body, self.policy, None)"},{"line_number":4153,"context_line":"        frag_archives \u003d ec_stub[\u0027frags\u0027]"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_af8f3773","line":4150,"updated":"2020-11-09 19:04:31.000000000","message":"ditto, unused ...","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9321099f12d8497a6736025f98b56c1886b1de2f","unresolved":false,"context_lines":[{"line_number":4208,"context_line":"    def test_GET_with_multirange_short_resume_body(self):"},{"line_number":4209,"context_line":"        self.app.object_chunk_size \u003d 256"},{"line_number":4210,"context_line":"        self.app.recoverable_node_timeout \u003d 0.01"},{"line_number":4211,"context_line":"        ec_stub \u003d self._make_ec_object_stub()"},{"line_number":4212,"context_line":"        test_body \u003d b\u0027test\u0027 * self.policy.ec_segment_size"},{"line_number":4213,"context_line":"        ec_stub \u003d make_ec_object_stub(test_body, self.policy, None)"},{"line_number":4214,"context_line":"        frag_archives \u003d ec_stub[\u0027frags\u0027]"}],"source_content_type":"text/x-python","patch_set":2,"id":"1f621f24_efeacf0c","line":4211,"updated":"2020-11-09 19:04:31.000000000","message":"ditto, unused...","commit_id":"34ee960bc51211728796ede4accb0fd76b01944b"}]}