)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When a backend GET times out, the proxy may try to resume the GET from"},{"line_number":10,"context_line":"a different backend source. If all retries fail, a ChunkReadTimeout"},{"line_number":11,"context_line":"may be raised."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"In the single range response case, this ChunkReadTimeout was not being"},{"line_number":14,"context_line":"caught, resulting in a traceback."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3a2070ca_621ad082","line":11,"updated":"2023-10-03 20:26:01.000000000","message":"I think we\u0027ve talked about how a NodesExhasutedError could make this more readable.","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When a backend GET times out, the proxy may try to resume the GET from"},{"line_number":10,"context_line":"a different backend source. If all retries fail, a ChunkReadTimeout"},{"line_number":11,"context_line":"may be raised."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"In the single range response case, this ChunkReadTimeout was not being"},{"line_number":14,"context_line":"caught, resulting in a traceback."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"c9e0b238_8aefbe4a","line":11,"in_reply_to":"3a2070ca_621ad082","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":11,"context_line":"may be raised."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"In the single range response case, this ChunkReadTimeout was not being"},{"line_number":14,"context_line":"caught, resulting in a traceback."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"In the multi-range repsonse case, the ChunkReadTimout was transformed"},{"line_number":17,"context_line":"to a StopIteration while trying to iterate over the response"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"50a16c5c_698e39c4","line":14,"updated":"2023-10-03 20:26:01.000000000","message":"probably an detailed error log message w/o a traceback would be sufficient for SRE","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":11,"context_line":"may be raised."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"In the single range response case, this ChunkReadTimeout was not being"},{"line_number":14,"context_line":"caught, resulting in a traceback."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"In the multi-range repsonse case, the ChunkReadTimout was transformed"},{"line_number":17,"context_line":"to a StopIteration while trying to iterate over the response"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"8e78fb76_8a99986d","line":14,"in_reply_to":"50a16c5c_698e39c4","updated":"2023-10-04 15:42:25.000000000","message":"Ack","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":17,"context_line":"to a StopIteration while trying to iterate over the response"},{"line_number":18,"context_line":"parts. This appeared the same as all parts being complete, and could"},{"line_number":19,"context_line":"result in the client receiving no parts but still getting a multi-part"},{"line_number":20,"context_line":"terminator line."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"This patch moves ChunkReadTimeout handling to a wrapper for the GET"},{"line_number":23,"context_line":"response app_iter. This has two benefits:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"713dde0c_a3e55977","line":20,"updated":"2023-10-03 20:26:01.000000000","message":"bah gross!","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":17,"context_line":"to a StopIteration while trying to iterate over the response"},{"line_number":18,"context_line":"parts. This appeared the same as all parts being complete, and could"},{"line_number":19,"context_line":"result in the client receiving no parts but still getting a multi-part"},{"line_number":20,"context_line":"terminator line."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"This patch moves ChunkReadTimeout handling to a wrapper for the GET"},{"line_number":23,"context_line":"response app_iter. This has two benefits:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"60945a79_990fdad6","line":20,"in_reply_to":"713dde0c_a3e55977","updated":"2023-10-04 15:42:25.000000000","message":"Ack","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"2. The exception causes the multipart response handler"},{"line_number":28,"context_line":"document_iters_to_multipart_byteranges to exit without yielding a"},{"line_number":29,"context_line":"terminator line."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In both cases the new handler ensures that the"},{"line_number":32,"context_line":"\u0027swift.non_client_disconnect\u0027 flag is set on the request environ."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"887eb851_b6ed458b","line":29,"updated":"2023-10-03 20:26:01.000000000","message":"is terminator ~\u003d boundary?  I thought it was the *first* line of the multipart document that would always come out even if we couldn\u0027t download anything from the backend?","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"2. The exception causes the multipart response handler"},{"line_number":28,"context_line":"document_iters_to_multipart_byteranges to exit without yielding a"},{"line_number":29,"context_line":"terminator line."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In both cases the new handler ensures that the"},{"line_number":32,"context_line":"\u0027swift.non_client_disconnect\u0027 flag is set on the request environ."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"d6b9f708_50ed6ca7","line":29,"in_reply_to":"887eb851_b6ed458b","updated":"2023-10-04 15:42:25.000000000","message":"yes, terminator ~\u003d boundary\n\nOn master it is possible that we yield it without previously yielding a boundary or part (because the StopIteration causes the part iter to exit cleanly):\nhttps://github.com/openstack/swift/blob/00bfc425ce3b1bfee38cea97be2ad0a734e4aa47/swift/common/utils/__init__.py#L4249","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":29,"context_line":"terminator line."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In both cases the new handler ensures that the"},{"line_number":32,"context_line":"\u0027swift.non_client_disconnect\u0027 flag is set on the request environ."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"For the single part case, exception logging is added."},{"line_number":35,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"e40ad009_1bea5885","line":32,"updated":"2023-10-03 20:26:01.000000000","message":"is this still needed/desirable?","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":29,"context_line":"terminator line."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In both cases the new handler ensures that the"},{"line_number":32,"context_line":"\u0027swift.non_client_disconnect\u0027 flag is set on the request environ."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"For the single part case, exception logging is added."},{"line_number":35,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5eb4d461_88921ae5","line":32,"in_reply_to":"e40ad009_1bea5885","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":7,"context_line":"proxy: log ChunkReadTimeouts in replicated GET path"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Add exception logging in more places where a ChunkReadTimeout is"},{"line_number":10,"context_line":"raised."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Some GetOrHeadHandler unit tests are moved to a new"},{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"f3fc811b_e67f427a","line":10,"updated":"2023-10-31 00:31:48.000000000","message":"and node error_increment; right?","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":7,"context_line":"proxy: log ChunkReadTimeouts in replicated GET path"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Add exception logging in more places where a ChunkReadTimeout is"},{"line_number":10,"context_line":"raised."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Some GetOrHeadHandler unit tests are moved to a new"},{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"e3461dc6_28e21b4e","line":10,"in_reply_to":"f3fc811b_e67f427a","updated":"2023-11-02 18:30:49.000000000","message":"eek! of course, dripping in an exception_occurred has that side-effect :/","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":10,"context_line":"raised."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Some GetOrHeadHandler unit tests are moved to a new"},{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Note: since the Related-Change, these timeouts will be handled by"},{"line_number":16,"context_line":"CatchErrorsMiddleware."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"4a918d85_67ee2ea2","line":13,"updated":"2023-10-31 00:31:48.000000000","message":"this is fine; but how is it related to the new logging?","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":10,"context_line":"raised."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Some GetOrHeadHandler unit tests are moved to a new"},{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Note: since the Related-Change, these timeouts will be handled by"},{"line_number":16,"context_line":"CatchErrorsMiddleware."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"6a067936_b1b59ac0","line":13,"in_reply_to":"4a918d85_67ee2ea2","updated":"2023-11-02 18:30:49.000000000","message":"Done","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Note: since the Related-Change, these timeouts will be handled by"},{"line_number":16,"context_line":"CatchErrorsMiddleware."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Related-Change: I280f84c07ec980cfb5246613ec522e6527aaaf29"},{"line_number":19,"context_line":"Change-Id: I403ba542834a8dcbe1052ea3c840453fcf358617"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"98195b54_6daf97cc","line":16,"updated":"2023-10-31 00:31:48.000000000","message":"where as before they just shot up all the way through eventlet.wsgi; not sure how that\u0027s related to what\u0027s going on here.","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":13,"context_line":"test_base.TestGetOrHeadHandler class, where new tests are also added."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Note: since the Related-Change, these timeouts will be handled by"},{"line_number":16,"context_line":"CatchErrorsMiddleware."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Related-Change: I280f84c07ec980cfb5246613ec522e6527aaaf29"},{"line_number":19,"context_line":"Change-Id: I403ba542834a8dcbe1052ea3c840453fcf358617"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"a8f8d7d4_2b91e8a5","line":16,"in_reply_to":"98195b54_6daf97cc","updated":"2023-11-02 18:30:49.000000000","message":"I anticipated readers might be curious as to what happens to these Timeouts. But it\u0027s moot now - I moved that change out of the dependency chain so cannot refer to it.","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"113ec620_02f27d7d","updated":"2023-10-03 20:26:01.000000000","message":"this is good!  squash this too!","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"e86b7927_927bba11","updated":"2023-10-04 15:42:25.000000000","message":"I intend to squash down but there\u0027s a tactical benefit to keep the non_client_disconnect fix out of the parent while we have chance to consider https://review.opendev.org/c/openstack/swift/+/897343","commit_id":"baa2941c5c176825bc86f2373e6d2003f9efdf67"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce037076b1c4040dcc89057a667553083699e584","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"2c5668a8_2ad6fbc2","updated":"2023-10-09 11:04:05.000000000","message":"\u003eI worry re-arraning all this to the front is trying to keep the refactor \"pure\" at the expense of grockability\n\nI\u0027m surprised if this is harder to grok than one patch with multiple bug fixes and code relocation.\n\nThis patch doesn\u0027t really enable the de-dup refactor. TestGetOrHeadHandler.test_node_timeout does evolve a bit in the patch chain, which makes it helpful to stack the patches.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"38e23638_6965907e","updated":"2023-10-06 19:27:27.000000000","message":"I think the test work looks like an improvement; but I don\u0027t love the fix!\n\nthose set_http_connect tests that some times leave comments about the acct, cont requests and some times look like they think they\u0027re testing object responses and sort of pass accidently are so annoying.  I though there was a pattern that pre-fills the cache to avoid the extra http requests but I couldn\u0027t find it.\n\nI worry re-arraning all this to the front is trying to keep the refactor \"pure\" at the expense of grockability.  I *think* I\u0027d prefer a diff \"was like this; total crap :RED: now it\u0027s lke this; tons better :GREEN: oh, and a whole bunch of tests to prove I thought about this real hard.\"\n\nIf we all agree it\u0027s better; we\u0027ll ship it - and if we break it; we\u0027ll fix it!  Who is the \"pure no behavior change at all costs\" refactor for?  Tried to make this code more understandable and fixed a bunch of annoyances along the way\" is a super reasonable commit message?  Squash it!","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"ebd798f7_ad1b1d68","updated":"2023-10-31 00:31:48.000000000","message":"I think it\u0027s maybe worth calling out the error_increment behavior in the commit message; possibly even sanity asserts in tests.\n\nwith the changes reverted we have some new asserts in the existing/updated proxy.test_server tests about the (new) missing log messages:\n\n\tFAILED swift/test/unit/proxy/test_server.py::TestReplicatedObjectController::test_node_read_timeout_retry - AssertionError: 4 !\u003d 3\n\tFAILED swift/test/unit/proxy/test_server.py::TestContainerController::test_node_read_timeout_retry_to_container - AssertionError: 1 !\u003d 0\n\nI feel like moving tests around *while* you\u0027re changing behavior isn\u0027t ideal; maybe re-org/re-write some tests (if you need to) and THEN fix-logging/add-error-counting or just add the logging/error-counting and fixup tests in place as much as possilbe (then save the test re-org for later)?  Maybe it doesn\u0027t matter; I just got a bit mixed up with which lines in diff were asserting new behaviors and which ones were just refactor/reorg test infra.  FWIW it seems like you can add the logging w/o breaking any existing tests - but maybe you were already refactoring tests when you realized you wanted to add the logging so maybe this diff is just the most convienient.  It\u0027s NBD for this specific change since 1) the tests_server timeout tests were kinda crap 2) the change is \"just\" new logging.","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"74e919ba2354b40d7658be32dfeff52e8200f282","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"b9f8ae3e_49306dea","updated":"2023-11-03 14:38:51.000000000","message":"WIP. The test failure for some python versions is odd.\n\n./test/unit/proxy/controllers/test_obj.py::TestECObjController::test_GET_read_timeout fails intermittently because not all the reads are timing out, but they *are* slow. I tried increasing the slowness and the test still fails sometimes.\n\nI got as far as convincing myself that the ChunkReadTimeout just doesn\u0027t pop sometimes. I noticed that, when resuming, when zero time elapses between the WatchdogTimeout in _get_next_reponse_part being created and the WatchdogTimeout in iter_bytes_from_response_part being created then they use the same key in the Watchdog, which seems flakey, bit I still can\u0027t see a path for one timeout to unset the key for the other. Unless there is some strange deferral of when the ocntext managers actually exit now that I have added a finally to the exception handler in iter_bytes_from_response_part.\n\nI\u0027ll need to dig more.","commit_id":"9104e8ae77f101f3d7b83a18a5de22a4db5dac79"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1339,"context_line":"                else:"},{"line_number":1340,"context_line":"                    self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":1341,"context_line":"                                                \u0027Trying to read during GET\u0027)"},{"line_number":1342,"context_line":"                    six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":1343,"context_line":"            else:"},{"line_number":1344,"context_line":"                if not chunk:"},{"line_number":1345,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":4,"id":"0460da43_869b30d2","line":1342,"updated":"2023-10-06 19:27:27.000000000","message":"how about we only re-raise a ShortReatError if that\u0027s the behavior we want/","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":true,"context_lines":[{"line_number":1339,"context_line":"                else:"},{"line_number":1340,"context_line":"                    self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":1341,"context_line":"                                                \u0027Trying to read during GET\u0027)"},{"line_number":1342,"context_line":"                    six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":1343,"context_line":"            else:"},{"line_number":1344,"context_line":"                if not chunk:"},{"line_number":1345,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":4,"id":"97ecfbef_6d2ee3fe","line":1342,"in_reply_to":"0460da43_869b30d2","updated":"2023-10-09 10:46:58.000000000","message":"I think I need to better understand the behaviour we want! Seems like we do need to raise *something* out of the proxy for wsgi to catch?","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":1339,"context_line":"                else:"},{"line_number":1340,"context_line":"                    self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":1341,"context_line":"                                                \u0027Trying to read during GET\u0027)"},{"line_number":1342,"context_line":"                    six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":1343,"context_line":"            else:"},{"line_number":1344,"context_line":"                if not chunk:"},{"line_number":1345,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":4,"id":"5c9ac99a_0904121b","line":1342,"in_reply_to":"56777bac_f8cc8bc8","updated":"2023-11-02 18:30:49.000000000","message":"thank you for reminding me of that!","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6fd97a1f794bfec35651cf06e0dc7b1a5fff6b87","unresolved":false,"context_lines":[{"line_number":1339,"context_line":"                else:"},{"line_number":1340,"context_line":"                    self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":1341,"context_line":"                                                \u0027Trying to read during GET\u0027)"},{"line_number":1342,"context_line":"                    six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":1343,"context_line":"            else:"},{"line_number":1344,"context_line":"                if not chunk:"},{"line_number":1345,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":4,"id":"8665cde2_a1a00ee8","line":1342,"in_reply_to":"5c9ac99a_0904121b","updated":"2023-12-04 18:52:26.000000000","message":"Done","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":1339,"context_line":"                else:"},{"line_number":1340,"context_line":"                    self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":1341,"context_line":"                                                \u0027Trying to read during GET\u0027)"},{"line_number":1342,"context_line":"                    six.reraise(exc_type, exc_value, exc_traceback)"},{"line_number":1343,"context_line":"            else:"},{"line_number":1344,"context_line":"                if not chunk:"},{"line_number":1345,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":4,"id":"56777bac_f8cc8bc8","line":1342,"in_reply_to":"97ecfbef_6d2ee3fe","updated":"2023-10-31 00:31:48.000000000","message":"so exception_occurred both logs (w/ \"exception\" method) and then also _error_increment\u0027s the node.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1614,"context_line":"                for chunk in it:"},{"line_number":1615,"context_line":"                    yield chunk"},{"line_number":1616,"context_line":"            except ChunkReadTimeout:"},{"line_number":1617,"context_line":"                pass"},{"line_number":1618,"context_line":""},{"line_number":1619,"context_line":"        return timeout_catcher(document_iters_to_http_response_body("},{"line_number":1620,"context_line":"            (add_content_type(pi) for pi in parts_iter),"}],"source_content_type":"text/x-python","patch_set":4,"id":"e121de2b_f27c8b78","line":1617,"updated":"2023-10-06 19:27:27.000000000","message":"this looks *really* stupid without the context of the above diff.\n\n    Errors should never pass silently.\n    Unless explicitly silenced.\n\nThe place to do the explit silence is right next to the place where we logged it.\n\nalso, if we\u0027ve learned anything shouldn\u0027t this wrapper close `it` in a finally block?","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":1614,"context_line":"                for chunk in it:"},{"line_number":1615,"context_line":"                    yield chunk"},{"line_number":1616,"context_line":"            except ChunkReadTimeout:"},{"line_number":1617,"context_line":"                pass"},{"line_number":1618,"context_line":""},{"line_number":1619,"context_line":"        return timeout_catcher(document_iters_to_http_response_body("},{"line_number":1620,"context_line":"            (add_content_type(pi) for pi in parts_iter),"}],"source_content_type":"text/x-python","patch_set":4,"id":"e88cdd64_3e281ca0","line":1617,"in_reply_to":"21fff7d7_52bd912f","updated":"2023-10-31 00:31:48.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc3401422f26afc3c45fdf80a3671d9d0fa9b59","unresolved":true,"context_lines":[{"line_number":1614,"context_line":"                for chunk in it:"},{"line_number":1615,"context_line":"                    yield chunk"},{"line_number":1616,"context_line":"            except ChunkReadTimeout:"},{"line_number":1617,"context_line":"                pass"},{"line_number":1618,"context_line":""},{"line_number":1619,"context_line":"        return timeout_catcher(document_iters_to_http_response_body("},{"line_number":1620,"context_line":"            (add_content_type(pi) for pi in parts_iter),"}],"source_content_type":"text/x-python","patch_set":4,"id":"21fff7d7_52bd912f","line":1617,"in_reply_to":"c4df02d0_36f4b5d6","updated":"2023-10-24 17:40:35.000000000","message":"I\u0027ve abandoned this part of the change in favour of doing something in catch_errors https://review.opendev.org/c/openstack/swift/+/899197/1?usp\u003drelated-change","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":true,"context_lines":[{"line_number":1614,"context_line":"                for chunk in it:"},{"line_number":1615,"context_line":"                    yield chunk"},{"line_number":1616,"context_line":"            except ChunkReadTimeout:"},{"line_number":1617,"context_line":"                pass"},{"line_number":1618,"context_line":""},{"line_number":1619,"context_line":"        return timeout_catcher(document_iters_to_http_response_body("},{"line_number":1620,"context_line":"            (add_content_type(pi) for pi in parts_iter),"}],"source_content_type":"text/x-python","patch_set":4,"id":"c4df02d0_36f4b5d6","line":1617,"in_reply_to":"e121de2b_f27c8b78","updated":"2023-10-09 10:46:58.000000000","message":"Yeah :/ and I\u0027d intended to mark this patch WIP (my bad!) because I wanted to ponder more if and what to log here. I think there is value in logging where the exception is raised because we get more specific context. But we still need to raise the exception for outer iters to catch. So do we annotate at source and log here, or log twice...or maybe (it seems) continue to not catch it here, or at least morph it to another exception for wsgi to catch.\n\n\u003e also, if we\u0027ve learned anything shouldn\u0027t this wrapper close it in a finally block?\n\nIt may be good hygiene, but I\u0027m no longer so sure what it guarantees when \u0027it\u0027 is, or wraps, a built-in generator, which AFAICT doesn\u0027t forward close() to whatever it wraps. We\u0027re actually relying everything wrapped inside the built-in being gc\u0027d.\n\n```\n# myiter raises exc\ninner \u003d (x for x in myiter)\nouter \u003d (y for y in inner)\ntry:\n    try:\n        next(outer)\n    except Exception as exc:\n        raise\n    finally:\n        inner.close() # this doesn\u0027t reach myiter\nexcept Exception as exc:\n    inner.close() # this doesn\u0027t reach myiter\n    print(\u0027caught again\u0027, exc)\n    # no more refs to exc so myiter get\u0027s gc\u0027d\n```\n\nIn this specific case (as written here, i.e. I\u0027m *not* raising the caught exception) the nest of iter\u0027s ought to be gc\u0027d in the except clause. But myiter.close() is not called.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":1108,"context_line":"                 path, backend_headers, logger\u003dNone):"},{"line_number":1109,"context_line":"        self.app \u003d app"},{"line_number":1110,"context_line":"        self.req \u003d req"},{"line_number":1111,"context_line":"        self.server_type \u003d server_type"},{"line_number":1112,"context_line":"        self.node_iter \u003d node_iter"},{"line_number":1113,"context_line":"        self.partition \u003d partition"},{"line_number":1114,"context_line":"        self.policy \u003d policy"}],"source_content_type":"text/x-python","patch_set":8,"id":"c3e375ef_4a735917","line":1111,"updated":"2023-11-02 18:45:33.000000000","message":"hmmm, this isn\u0027t strictly necessary to be an attribute of the base class - both subclasses use it to pass to app.exception_occurred, but ECFragGetter could just use the string \u0027Object\u0027 .\n\nIt is needed in GetterBase by the end of the patch chain.","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":1131,"context_line":"        raise NotImplementedError()"},{"line_number":1132,"context_line":""},{"line_number":1133,"context_line":"    def _replace_source(self):"},{"line_number":1134,"context_line":"        if self.source:"},{"line_number":1135,"context_line":"            self.source.close()"},{"line_number":1136,"context_line":"        return self._find_source()"},{"line_number":1137,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"d82e729f_36ee3071","line":1134,"updated":"2023-11-02 18:45:33.000000000","message":"I have pulled the error_occurred out of _replace_source - I never liked passing an error message in for a side-effect of this method, and the side effect was just going to get more bloated to differentiate \"retrying\" from \"no more nodes\"\n\nBut, others may have stuck with the existing pattern, and I dabbled with both, so let\u0027s see how the change is received.","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":1356,"context_line":"                finally:"},{"line_number":1357,"context_line":"                    self.app.exception_occurred("},{"line_number":1358,"context_line":"                        old_source.node, self.server_type,"},{"line_number":1359,"context_line":"                        \u0027Trying to read object during GET (%s)\u0027 % msg)"},{"line_number":1360,"context_line":"            else:"},{"line_number":1361,"context_line":"                if not chunk:"},{"line_number":1362,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":8,"id":"396c5f13_80a4d8ee","line":1359,"updated":"2023-11-02 18:45:33.000000000","message":"I\u0027m trying to achieve two things with this try/except/else/finally:\n\n1. log/increment the error in *all* subsequent scenarios cases\n2. Include appropriate msg in each scenario (so log in the finally)\n\nI played with a number of different approaches trying to condense this but with no success.","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":2636,"context_line":"        except ChunkReadTimeout:"},{"line_number":2637,"context_line":"            self.app.exception_occurred(self.source.node, \u0027Object\u0027,"},{"line_number":2638,"context_line":"                                        \u0027Trying to read during GET\u0027)"},{"line_number":2639,"context_line":"            raise"},{"line_number":2640,"context_line":"        except ChunkWriteTimeout:"},{"line_number":2641,"context_line":"            self.logger.warning("},{"line_number":2642,"context_line":"                \u0027Client did not read from proxy within %ss\u0027 %"}],"source_content_type":"text/x-python","patch_set":8,"id":"ebbf1a32_b24c5687","side":"PARENT","line":2639,"updated":"2023-11-02 18:45:33.000000000","message":"this was the cause of the double error increment bug - we already logged this ChunkReadTimeout in _get_next_response_part","commit_id":"2059b1a90fe1f06a7c1844561bbeaff95c4bc865"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":2592,"context_line":"                finally:"},{"line_number":2593,"context_line":"                    self.app.exception_occurred("},{"line_number":2594,"context_line":"                        old_source.node, self.server_type,"},{"line_number":2595,"context_line":"                        \u0027Trying to read EC fragment during GET (%s)\u0027 % msg)"},{"line_number":2596,"context_line":"            else:"},{"line_number":2597,"context_line":"                if buf and self.skip_bytes:"},{"line_number":2598,"context_line":"                    if self.skip_bytes \u003c len(buf):"}],"source_content_type":"text/x-python","patch_set":8,"id":"71a4002a_8f78e5bf","line":2595,"updated":"2023-11-02 18:45:33.000000000","message":"ditto comment in base.py - I couldn\u0027t find a way to condense this try/except/else/finally","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"876025b238dcb4b2399ad756dc1c7134ee5b72f3","unresolved":true,"context_lines":[{"line_number":2632,"context_line":"                    # except ChunkReadTimeout:"},{"line_number":2633,"context_line":"                    #   with a multipart response a ChunkReadTimeout may be"},{"line_number":2634,"context_line":"                    #   raised, but it\u0027s already been logged in"},{"line_number":2635,"context_line":"                    #   _get_next_response_part so no need to catch it here"},{"line_number":2636,"context_line":""},{"line_number":2637,"context_line":"                    # skip_bytes compensates for the backend request range"},{"line_number":2638,"context_line":"                    # expansion done in _convert_range"}],"source_content_type":"text/x-python","patch_set":8,"id":"20cb2225_b1b35d5b","line":2635,"updated":"2023-11-02 18:45:33.000000000","message":"I felt this comment was useful but I\u0027m not sure if the format will be to everyone\u0027s taste?","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"}],"test/unit/proxy/controllers/test_base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1750,"context_line":""},{"line_number":1751,"context_line":"@patch_policies([StoragePolicy(0, \u0027zero\u0027, True, object_ring\u003dFakeRing())])"},{"line_number":1752,"context_line":"class TestGetOrHeadHandler(BaseTest):"},{"line_number":1753,"context_line":"    def test_init_node_timeout(self):"},{"line_number":1754,"context_line":"        conf \u003d {\u0027node_timeout\u0027: 2, \u0027recoverable_node_timeout\u0027: 3}"},{"line_number":1755,"context_line":"        app \u003d proxy_server.Application(conf,"},{"line_number":1756,"context_line":"                                       logger\u003dself.logger,"}],"source_content_type":"text/x-python","patch_set":4,"id":"f6581c5a_b2555c14","line":1753,"updated":"2023-10-06 19:27:27.000000000","message":"this is a new test name I think","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":false,"context_lines":[{"line_number":1750,"context_line":""},{"line_number":1751,"context_line":"@patch_policies([StoragePolicy(0, \u0027zero\u0027, True, object_ring\u003dFakeRing())])"},{"line_number":1752,"context_line":"class TestGetOrHeadHandler(BaseTest):"},{"line_number":1753,"context_line":"    def test_init_node_timeout(self):"},{"line_number":1754,"context_line":"        conf \u003d {\u0027node_timeout\u0027: 2, \u0027recoverable_node_timeout\u0027: 3}"},{"line_number":1755,"context_line":"        app \u003d proxy_server.Application(conf,"},{"line_number":1756,"context_line":"                                       logger\u003dself.logger,"}],"source_content_type":"text/x-python","patch_set":4,"id":"7b690591_4b79130c","line":1753,"in_reply_to":"f6581c5a_b2555c14","updated":"2023-10-09 10:46:58.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1774,"context_line":"        self.app.logger \u003d mock.Mock()"},{"line_number":1775,"context_line":"        req \u003d Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":1776,"context_line":"        headers \u003d {\u0027content-type\u0027: \u0027text/plain\u0027}"},{"line_number":1777,"context_line":"        source \u003d FakeSource([], headers\u003dheaders, body\u003db\u0027the cake is a lie\u0027)"},{"line_number":1778,"context_line":""},{"line_number":1779,"context_line":"        node \u003d {\u0027ip\u0027: \u00271.2.3.4\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sda\u0027}"},{"line_number":1780,"context_line":"        handler \u003d GetOrHeadHandler("}],"source_content_type":"text/x-python","patch_set":4,"id":"fa186835_e478d1ba","line":1777,"updated":"2023-10-06 19:27:27.000000000","message":"seems like a faithful extraction, GLaDOS approves.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":false,"context_lines":[{"line_number":1774,"context_line":"        self.app.logger \u003d mock.Mock()"},{"line_number":1775,"context_line":"        req \u003d Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":1776,"context_line":"        headers \u003d {\u0027content-type\u0027: \u0027text/plain\u0027}"},{"line_number":1777,"context_line":"        source \u003d FakeSource([], headers\u003dheaders, body\u003db\u0027the cake is a lie\u0027)"},{"line_number":1778,"context_line":""},{"line_number":1779,"context_line":"        node \u003d {\u0027ip\u0027: \u00271.2.3.4\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sda\u0027}"},{"line_number":1780,"context_line":"        handler \u003d GetOrHeadHandler("}],"source_content_type":"text/x-python","patch_set":4,"id":"0e9570e7_a1a865bc","line":1777,"in_reply_to":"fa186835_e478d1ba","updated":"2023-10-09 10:46:58.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1805,"context_line":"            resp.app_iter.close()"},{"line_number":1806,"context_line":"        self.app.logger.warning.assert_not_called()"},{"line_number":1807,"context_line":""},{"line_number":1808,"context_line":"    def test_node_timeout(self):"},{"line_number":1809,"context_line":"        def do_test(server_type, req_headers):"},{"line_number":1810,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, headers\u003dreq_headers)"},{"line_number":1811,"context_line":"            resp_headers \u003d {\u0027content-type\u0027: \u0027text/plain\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"e3ee3616_b357b84e","line":1808,"updated":"2023-10-06 19:27:27.000000000","message":"this is a new test name i think","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":false,"context_lines":[{"line_number":1805,"context_line":"            resp.app_iter.close()"},{"line_number":1806,"context_line":"        self.app.logger.warning.assert_not_called()"},{"line_number":1807,"context_line":""},{"line_number":1808,"context_line":"    def test_node_timeout(self):"},{"line_number":1809,"context_line":"        def do_test(server_type, req_headers):"},{"line_number":1810,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, headers\u003dreq_headers)"},{"line_number":1811,"context_line":"            resp_headers \u003d {\u0027content-type\u0027: \u0027text/plain\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"92578463_ce28f318","line":1808,"in_reply_to":"e3ee3616_b357b84e","updated":"2023-10-09 10:46:58.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1865,"context_line":"            self.app, req, \u0027test\u0027, Namespace(num_primary_nodes\u003d3), None, None,"},{"line_number":1866,"context_line":"            {})"},{"line_number":1867,"context_line":"        handler.fast_forward(50)"},{"line_number":1868,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d50-\u0027)"},{"line_number":1869,"context_line":""},{"line_number":1870,"context_line":"        handler \u003d GetOrHeadHandler("},{"line_number":1871,"context_line":"            self.app, req, \u0027test\u0027, Namespace(num_primary_nodes\u003d3), None, None,"}],"source_content_type":"text/x-python","patch_set":4,"id":"3333c198_9b9b5ffd","line":1868,"updated":"2023-10-06 19:27:27.000000000","message":"non ranged request that does NOT get \"learn_size_from_content_range\" will do a backend request that\u0027s NOT capped (i.e. no 50- has no ending bytes).","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":1865,"context_line":"            self.app, req, \u0027test\u0027, Namespace(num_primary_nodes\u003d3), None, None,"},{"line_number":1866,"context_line":"            {})"},{"line_number":1867,"context_line":"        handler.fast_forward(50)"},{"line_number":1868,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d50-\u0027)"},{"line_number":1869,"context_line":""},{"line_number":1870,"context_line":"        handler \u003d GetOrHeadHandler("},{"line_number":1871,"context_line":"            self.app, req, \u0027test\u0027, Namespace(num_primary_nodes\u003d3), None, None,"}],"source_content_type":"text/x-python","patch_set":4,"id":"eff62488_d75aa589","line":1868,"in_reply_to":"3333c198_9b9b5ffd","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1889,"context_line":"        handler.fast_forward(20)"},{"line_number":1890,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d-80\u0027)"},{"line_number":1891,"context_line":"        self.assertRaises(HTTPException,"},{"line_number":1892,"context_line":"                          handler.fast_forward, 100)"},{"line_number":1893,"context_line":"        self.assertRaises(exceptions.RangeAlreadyComplete,"},{"line_number":1894,"context_line":"                          handler.fast_forward, 80)"},{"line_number":1895,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"eb8461d6_ef426258","line":1892,"updated":"2023-10-06 19:27:27.000000000","message":"this seems inline with the existing test, but I\u0027m not sure I understand it","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":1889,"context_line":"        handler.fast_forward(20)"},{"line_number":1890,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d-80\u0027)"},{"line_number":1891,"context_line":"        self.assertRaises(HTTPException,"},{"line_number":1892,"context_line":"                          handler.fast_forward, 100)"},{"line_number":1893,"context_line":"        self.assertRaises(exceptions.RangeAlreadyComplete,"},{"line_number":1894,"context_line":"                          handler.fast_forward, 80)"},{"line_number":1895,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"9caf10a4_96cf657a","line":1892,"in_reply_to":"eb8461d6_ef426258","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1909,"context_line":"        self.assertNotIn(\u0027X-Backend-Ignore-Range-If-Metadata-Present\u0027,"},{"line_number":1910,"context_line":"                         handler.backend_headers)"},{"line_number":1911,"context_line":""},{"line_number":1912,"context_line":"    def test_range_fast_forward_after_data_timeout(self):"},{"line_number":1913,"context_line":"        req \u003d Request.blank(\u0027/\u0027)"},{"line_number":1914,"context_line":""},{"line_number":1915,"context_line":"        # We get a 200 and learn that it\u0027s a 1000-byte object, but receive 0"}],"source_content_type":"text/x-python","patch_set":4,"id":"e98379cb_9a31f850","line":1912,"updated":"2023-10-06 19:27:27.000000000","message":"ok, it\u0027s basically tests for fast_forward(0); I probably don\u0027t understand why the tests for fast_forward(N) don\u0027t also call learn_size_from_content_range","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":1909,"context_line":"        self.assertNotIn(\u0027X-Backend-Ignore-Range-If-Metadata-Present\u0027,"},{"line_number":1910,"context_line":"                         handler.backend_headers)"},{"line_number":1911,"context_line":""},{"line_number":1912,"context_line":"    def test_range_fast_forward_after_data_timeout(self):"},{"line_number":1913,"context_line":"        req \u003d Request.blank(\u0027/\u0027)"},{"line_number":1914,"context_line":""},{"line_number":1915,"context_line":"        # We get a 200 and learn that it\u0027s a 1000-byte object, but receive 0"}],"source_content_type":"text/x-python","patch_set":4,"id":"1ff0e696_9f3463e1","line":1912,"in_reply_to":"e98379cb_9a31f850","updated":"2023-11-02 18:30:49.000000000","message":"existing test - now moved in a pre-req patch","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":1921,"context_line":"            {})"},{"line_number":1922,"context_line":"        handler.learn_size_from_content_range(0, 999, 1000)"},{"line_number":1923,"context_line":"        handler.fast_forward(0)"},{"line_number":1924,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d0-999\u0027)"},{"line_number":1925,"context_line":""},{"line_number":1926,"context_line":"        # Same story as above, but a 1-byte object so we can have our byte"},{"line_number":1927,"context_line":"        # indices be 0."}],"source_content_type":"text/x-python","patch_set":4,"id":"0092f2a8_aadebe54","line":1924,"updated":"2023-10-06 19:27:27.000000000","message":"non ranged request that gets \"learn_size_from_content_range\" gets a backend request that\u0027s capped.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":1921,"context_line":"            {})"},{"line_number":1922,"context_line":"        handler.learn_size_from_content_range(0, 999, 1000)"},{"line_number":1923,"context_line":"        handler.fast_forward(0)"},{"line_number":1924,"context_line":"        self.assertEqual(handler.backend_headers[\u0027Range\u0027], \u0027bytes\u003d0-999\u0027)"},{"line_number":1925,"context_line":""},{"line_number":1926,"context_line":"        # Same story as above, but a 1-byte object so we can have our byte"},{"line_number":1927,"context_line":"        # indices be 0."}],"source_content_type":"text/x-python","patch_set":4,"id":"1c64ad96_2a9bfeab","line":1924,"in_reply_to":"0092f2a8_aadebe54","updated":"2023-11-02 18:30:49.000000000","message":"existing test - now moved in a pre-req patch","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c7a9f6cfe2d0a67514ad3fa35ad4488de85a4664","unresolved":true,"context_lines":[{"line_number":1851,"context_line":"        lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1852,"context_line":"        self.assertIn(\u0027ERROR with Container server\u0027, lines[0])"},{"line_number":1853,"context_line":"        self.assertIn(\u0027Trying to read during GET: ChunkReadTimeout\u0027, lines[0])"},{"line_number":1854,"context_line":"        self.assertFalse(self.logger.get_lines_for_level(\u0027info\u0027))"},{"line_number":1855,"context_line":""},{"line_number":1856,"context_line":"        self.logger.clear()"},{"line_number":1857,"context_line":"        do_test(\u0027Account\u0027, {})"}],"source_content_type":"text/x-python","patch_set":5,"id":"382ccfb5_53857446","line":1854,"updated":"2023-10-19 18:56:27.000000000","message":"So on py2 we get\n```\nAssertionError: [\"Client disconnected on read of \u0027some-path\u0027\"] is not false\n```\nhere ... but I don\u0027t understand _why_. It _looks like_ we\u0027re just adding logging and translating `ChunkReadTimeout` -\u003e `StopIteration`, and (I thought) `StopIteration` should be setting the `non_client_disconnect` flag...","commit_id":"ef50cf4017dc174f1119f94be666d7887ff0dbaf"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":1804,"context_line":"            resp \u003d handler.get_working_response(req)"},{"line_number":1805,"context_line":"            next(resp.app_iter)"},{"line_number":1806,"context_line":"            resp.app_iter.close()"},{"line_number":1807,"context_line":"        self.app.logger.warning.assert_not_called()"},{"line_number":1808,"context_line":""},{"line_number":1809,"context_line":"    def test_node_timeout(self):"},{"line_number":1810,"context_line":"        def do_test(server_type, req_headers):"}],"source_content_type":"text/x-python","patch_set":7,"id":"89864740_322f6cbd","line":1807,"updated":"2023-10-31 00:31:48.000000000","message":"whatever logging this test was asserting on before it was moved still seems to pass with the new logging removed","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":1804,"context_line":"            resp \u003d handler.get_working_response(req)"},{"line_number":1805,"context_line":"            next(resp.app_iter)"},{"line_number":1806,"context_line":"            resp.app_iter.close()"},{"line_number":1807,"context_line":"        self.app.logger.warning.assert_not_called()"},{"line_number":1808,"context_line":""},{"line_number":1809,"context_line":"    def test_node_timeout(self):"},{"line_number":1810,"context_line":"        def do_test(server_type, req_headers):"}],"source_content_type":"text/x-python","patch_set":7,"id":"66e07afa_4b72893f","line":1807,"in_reply_to":"89864740_322f6cbd","updated":"2023-11-02 18:30:49.000000000","message":"this test was only relocated - now in pre-req patch","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":1833,"context_line":"        # for the object server type the handler retries before raising timeout"},{"line_number":1834,"context_line":"        do_test(\u0027Object\u0027, {})"},{"line_number":1835,"context_line":"        lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1836,"context_line":"        self.assertEqual(3, len(lines))"},{"line_number":1837,"context_line":"        self.assertIn(\u0027Trying to read object during GET (retrying)\u0027, lines[0])"},{"line_number":1838,"context_line":"        self.assertIn(\u0027Trying to read object during GET (retrying)\u0027, lines[1])"},{"line_number":1839,"context_line":"        self.assertIn(\u0027ERROR with Object server\u0027, lines[2])"}],"source_content_type":"text/x-python","patch_set":7,"id":"9d85e424_d1944872","line":1836,"updated":"2023-10-31 00:31:48.000000000","message":"I think this is an entirely new test, it also fails with the new logging removed\n\nFAILED swift/test/unit/proxy/controllers/test_base.py::TestGetOrHeadHandler::test_node_timeout - AssertionError: 3 !\u003d 2","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":1833,"context_line":"        # for the object server type the handler retries before raising timeout"},{"line_number":1834,"context_line":"        do_test(\u0027Object\u0027, {})"},{"line_number":1835,"context_line":"        lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1836,"context_line":"        self.assertEqual(3, len(lines))"},{"line_number":1837,"context_line":"        self.assertIn(\u0027Trying to read object during GET (retrying)\u0027, lines[0])"},{"line_number":1838,"context_line":"        self.assertIn(\u0027Trying to read object during GET (retrying)\u0027, lines[1])"},{"line_number":1839,"context_line":"        self.assertIn(\u0027ERROR with Object server\u0027, lines[2])"}],"source_content_type":"text/x-python","patch_set":7,"id":"493e3cd2_eacbbb63","line":1836,"in_reply_to":"9d85e424_d1944872","updated":"2023-11-02 18:30:49.000000000","message":"Ack","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a92fce2019096b051677f05cb321d85b1de1d325","unresolved":true,"context_lines":[{"line_number":1757,"context_line":"                      {\u0027ip\u0027: \u00271.2.3.5\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sdb\u0027},"},{"line_number":1758,"context_line":"                      {\u0027ip\u0027: \u00271.2.3.6\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sdc\u0027}]"},{"line_number":1759,"context_line":""},{"line_number":1760,"context_line":"    def test_init_node_timeout(self):"},{"line_number":1761,"context_line":"        conf \u003d {\u0027node_timeout\u0027: 2, \u0027recoverable_node_timeout\u0027: 3}"},{"line_number":1762,"context_line":"        app \u003d proxy_server.Application(conf,"},{"line_number":1763,"context_line":"                                       logger\u003dself.logger,"}],"source_content_type":"text/x-python","patch_set":8,"id":"0241e415_12f42112","line":1760,"updated":"2023-11-02 18:49:40.000000000","message":"hmm, I guess this could be in the parent patch","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6fd97a1f794bfec35651cf06e0dc7b1a5fff6b87","unresolved":false,"context_lines":[{"line_number":1757,"context_line":"                      {\u0027ip\u0027: \u00271.2.3.5\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sdb\u0027},"},{"line_number":1758,"context_line":"                      {\u0027ip\u0027: \u00271.2.3.6\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027sdc\u0027}]"},{"line_number":1759,"context_line":""},{"line_number":1760,"context_line":"    def test_init_node_timeout(self):"},{"line_number":1761,"context_line":"        conf \u003d {\u0027node_timeout\u0027: 2, \u0027recoverable_node_timeout\u0027: 3}"},{"line_number":1762,"context_line":"        app \u003d proxy_server.Application(conf,"},{"line_number":1763,"context_line":"                                       logger\u003dself.logger,"}],"source_content_type":"text/x-python","patch_set":8,"id":"d3df23b8_de347ca2","line":1760,"in_reply_to":"0241e415_12f42112","updated":"2023-12-04 18:52:26.000000000","message":"Done","commit_id":"c0adc2cf30113b3aec87b6c46c5824f84b9d0948"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":1781,"context_line":"        self.assertTrue(req.environ.get(\u0027swift.non_client_disconnect\u0027))"},{"line_number":1782,"context_line":"        self.assertEqual(6, len(log))"},{"line_number":1783,"context_line":"        resp_boundary \u003d resp.headers[\u0027content-type\u0027].rsplit(\u0027\u003d\u0027, 1)[1].encode()"},{"line_number":1784,"context_line":"        self.assertEqual(b\u0027--%s--\u0027 % resp_boundary, actual_body)"},{"line_number":1785,"context_line":"        error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1786,"context_line":"        self.assertEqual(3, len(error_lines))"},{"line_number":1787,"context_line":"        for line in error_lines:"}],"source_content_type":"text/x-python","patch_set":2,"id":"95686be6_c572ce8c","side":"PARENT","line":1784,"updated":"2023-10-03 20:26:01.000000000","message":"oh, getting rid of this is nice actually!?","commit_id":"cb8a0b497b330fa68ec1d9859813acb00ba75753"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":1781,"context_line":"        self.assertTrue(req.environ.get(\u0027swift.non_client_disconnect\u0027))"},{"line_number":1782,"context_line":"        self.assertEqual(6, len(log))"},{"line_number":1783,"context_line":"        resp_boundary \u003d resp.headers[\u0027content-type\u0027].rsplit(\u0027\u003d\u0027, 1)[1].encode()"},{"line_number":1784,"context_line":"        self.assertEqual(b\u0027--%s--\u0027 % resp_boundary, actual_body)"},{"line_number":1785,"context_line":"        error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1786,"context_line":"        self.assertEqual(3, len(error_lines))"},{"line_number":1787,"context_line":"        for line in error_lines:"}],"source_content_type":"text/x-python","patch_set":2,"id":"caa950c2_1bc9fda9","side":"PARENT","line":1784,"in_reply_to":"95686be6_c572ce8c","updated":"2023-10-04 15:42:25.000000000","message":"Ack","commit_id":"cb8a0b497b330fa68ec1d9859813acb00ba75753"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":1777,"context_line":"            self.assertEqual(resp.status_int, 206)"},{"line_number":1778,"context_line":"            actual_body \u003d resp.body"},{"line_number":1779,"context_line":""},{"line_number":1780,"context_line":"        self.assertEqual(resp.status_int, 206)"},{"line_number":1781,"context_line":"        self.assertTrue(req.environ.get(\u0027swift.non_client_disconnect\u0027))"},{"line_number":1782,"context_line":"        self.assertEqual(6, len(log))"},{"line_number":1783,"context_line":"        self.assertEqual(b\u0027\u0027, actual_body)"}],"source_content_type":"text/x-python","patch_set":2,"id":"a19fe10f_740595f2","line":1780,"updated":"2023-10-03 20:26:01.000000000","message":"it sounds like maybe there is a behavior difference from EC vs Replicated in this case?  Does EC manage to get out a 5XX?","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":1777,"context_line":"            self.assertEqual(resp.status_int, 206)"},{"line_number":1778,"context_line":"            actual_body \u003d resp.body"},{"line_number":1779,"context_line":""},{"line_number":1780,"context_line":"        self.assertEqual(resp.status_int, 206)"},{"line_number":1781,"context_line":"        self.assertTrue(req.environ.get(\u0027swift.non_client_disconnect\u0027))"},{"line_number":1782,"context_line":"        self.assertEqual(6, len(log))"},{"line_number":1783,"context_line":"        self.assertEqual(b\u0027\u0027, actual_body)"}],"source_content_type":"text/x-python","patch_set":2,"id":"2bd93b14_fe0fa7ac","line":1780,"in_reply_to":"a19fe10f_740595f2","updated":"2023-10-04 15:42:25.000000000","message":"yes test.unit.proxy.controllers.test_obj.TestECObjController.test_GET_with_multirange_unable_to_resume\n\nMaybe we can get the replicated path to do the same, but I\u0027d like to defer that to a follow-up - it\u0027s not obvious to me where to do the surgery.","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":1787,"context_line":"            self.assertIn(\u0027Trying to read next part of object multi-part GET\u0027,"},{"line_number":1788,"context_line":"                          line)"},{"line_number":1789,"context_line":"        self.assertIn(\u0027Trying to read during GET: ChunkReadTimeout \u0027,"},{"line_number":1790,"context_line":"                      error_lines[3])"},{"line_number":1791,"context_line":""},{"line_number":1792,"context_line":"    def test_GET_unable_to_resume(self):"},{"line_number":1793,"context_line":"        self.app.recoverable_node_timeout \u003d 0.01"}],"source_content_type":"text/x-python","patch_set":2,"id":"42cfbe19_58a688e7","line":1790,"updated":"2023-10-03 20:26:01.000000000","message":"final RIP log line is nice too","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":false,"context_lines":[{"line_number":1787,"context_line":"            self.assertIn(\u0027Trying to read next part of object multi-part GET\u0027,"},{"line_number":1788,"context_line":"                          line)"},{"line_number":1789,"context_line":"        self.assertIn(\u0027Trying to read during GET: ChunkReadTimeout \u0027,"},{"line_number":1790,"context_line":"                      error_lines[3])"},{"line_number":1791,"context_line":""},{"line_number":1792,"context_line":"    def test_GET_unable_to_resume(self):"},{"line_number":1793,"context_line":"        self.app.recoverable_node_timeout \u003d 0.01"}],"source_content_type":"text/x-python","patch_set":2,"id":"1b9e0fff_2485e275","line":1790,"in_reply_to":"42cfbe19_58a688e7","updated":"2023-10-04 15:42:25.000000000","message":"Ack","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":1813,"context_line":"            return responses.pop(0) if responses else StubResponse(404)"},{"line_number":1814,"context_line":""},{"line_number":1815,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":1816,"context_line":"        with capture_http_requests(get_response):"},{"line_number":1817,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":1818,"context_line":"            actual_body \u003d resp.body"},{"line_number":1819,"context_line":"        self.assertEqual(resp.status_int, 200)"}],"source_content_type":"text/x-python","patch_set":2,"id":"05a56f53_5efcfbb6","line":1816,"updated":"2023-10-03 20:26:01.000000000","message":"i think i\u0027m more used to tests that use mocked_http_connection \u0026 FakeStatus with response_sleep - but those seem to pass in separte body_iter\n\ntest.unit.StubResponse has the advantage of keeping status/body/headers together on a single object, but requires the extra get_response callable and the in-module capture_http_requests (very similar to test.unit.fake_http_connect).","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":1813,"context_line":"            return responses.pop(0) if responses else StubResponse(404)"},{"line_number":1814,"context_line":""},{"line_number":1815,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":1816,"context_line":"        with capture_http_requests(get_response):"},{"line_number":1817,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":1818,"context_line":"            actual_body \u003d resp.body"},{"line_number":1819,"context_line":"        self.assertEqual(resp.status_int, 200)"}],"source_content_type":"text/x-python","patch_set":2,"id":"39ee66e2_da5d9889","line":1816,"in_reply_to":"05a56f53_5efcfbb6","updated":"2023-10-04 15:42:25.000000000","message":"ack - if I get time I\u0027ll refactor to fake_http_connect","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":1819,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":1820,"context_line":"        self.assertEqual(b\u0027\u0027, actual_body)"},{"line_number":1821,"context_line":"        self.assertEqual(etag, resp.headers.get(\u0027ETag\u0027))"},{"line_number":1822,"context_line":"        self.assertTrue(req.environ.get(\u0027swift.non_client_disconnect\u0027))"},{"line_number":1823,"context_line":"        error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":1824,"context_line":"        self.assertEqual(4, len(error_lines))"},{"line_number":1825,"context_line":"        for line in error_lines[:3]:"}],"source_content_type":"text/x-python","patch_set":2,"id":"bc5eb34c_75f47bc6","line":1822,"updated":"2023-10-04 15:42:25.000000000","message":"this fails if we don\u0027t set swift.non_client_disconnect in timeout_catcher","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":5351,"context_line":"                got_exc \u003d True"},{"line_number":5352,"context_line":"            self.assertFalse(got_exc)"},{"line_number":5353,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5354,"context_line":"            set_http_connect(200, 200, 200, slow\u003d1.0)"},{"line_number":5355,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5356,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5357,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"95eecefe_15a63655","line":5354,"updated":"2023-10-04 15:42:25.000000000","message":"what\u0027s actually going one here? we\u0027re re-using the same request, which now has account and container info in its infocache, so all the response codes are for object requests !?!","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":5351,"context_line":"                got_exc \u003d True"},{"line_number":5352,"context_line":"            self.assertFalse(got_exc)"},{"line_number":5353,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5354,"context_line":"            set_http_connect(200, 200, 200, slow\u003d1.0)"},{"line_number":5355,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5356,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5357,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"f8ca158e_af18dd8a","line":5354,"in_reply_to":"95eecefe_15a63655","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":5355,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5356,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5357,"context_line":""},{"line_number":5358,"context_line":"    def test_node_read_timeout_retry(self):"},{"line_number":5359,"context_line":"        with save_globals():"},{"line_number":5360,"context_line":"            self.app.account_ring.get_nodes(\u0027account\u0027)"},{"line_number":5361,"context_line":"            for dev in self.app.account_ring.devs:"}],"source_content_type":"text/x-python","patch_set":2,"id":"f1e43f16_34fcbba8","line":5358,"updated":"2023-10-04 15:42:25.000000000","message":"this test seems to be a superset of the previous?","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":5355,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5356,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5357,"context_line":""},{"line_number":5358,"context_line":"    def test_node_read_timeout_retry(self):"},{"line_number":5359,"context_line":"        with save_globals():"},{"line_number":5360,"context_line":"            self.app.account_ring.get_nodes(\u0027account\u0027)"},{"line_number":5361,"context_line":"            for dev in self.app.account_ring.devs:"}],"source_content_type":"text/x-python","patch_set":2,"id":"e2d76444_ef858e5d","line":5358,"in_reply_to":"f1e43f16_34fcbba8","updated":"2023-11-02 18:30:49.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":5371,"context_line":"                dev[\u0027ip\u0027] \u003d \u0027127.0.0.1\u0027"},{"line_number":5372,"context_line":"                dev[\u0027port\u0027] \u003d 1"},{"line_number":5373,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{"},{"line_number":5374,"context_line":"                \u0027REQUEST_METHOD\u0027: \u0027GET\u0027, \u0027swift.cache\u0027: FakeMemcache()})"},{"line_number":5375,"context_line":"            self.app.update_request(req)"},{"line_number":5376,"context_line":""},{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"}],"source_content_type":"text/x-python","patch_set":2,"id":"46a1ffcf_8b17a859","line":5374,"updated":"2023-10-04 15:42:25.000000000","message":"setup a fake memcache AND keep reusing the same request??","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":5371,"context_line":"                dev[\u0027ip\u0027] \u003d \u0027127.0.0.1\u0027"},{"line_number":5372,"context_line":"                dev[\u0027port\u0027] \u003d 1"},{"line_number":5373,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{"},{"line_number":5374,"context_line":"                \u0027REQUEST_METHOD\u0027: \u0027GET\u0027, \u0027swift.cache\u0027: FakeMemcache()})"},{"line_number":5375,"context_line":"            self.app.update_request(req)"},{"line_number":5376,"context_line":""},{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"}],"source_content_type":"text/x-python","patch_set":2,"id":"457176de_a0148199","line":5374,"in_reply_to":"46a1ffcf_8b17a859","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7a2866d17bfd783cb8aa0b71c80e8cd5e58ce97d","unresolved":true,"context_lines":[{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5378,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5379,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5380,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"7d56bcd0_2bc99776","line":5380,"updated":"2023-10-03 20:26:01.000000000","message":"ok, but maybe assert logging still has some kind of error?  is the status code here success?","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5378,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5379,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5380,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5a6629_2844667f","line":5380,"in_reply_to":"56872f51_6cc76aa5","updated":"2023-10-06 19:27:27.000000000","message":"logging assertions looks solid; but I still don\u0027t know what the response code is - NM, it\u0027s probably 200 same as the others that blew up on the body.","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5378,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5379,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5380,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"56872f51_6cc76aa5","line":5380,"in_reply_to":"7d56bcd0_2bc99776","updated":"2023-10-04 15:42:25.000000000","message":"fair point - these tests burnt me late in the evening and I rushed to fix :/","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":5377,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5378,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5379,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5380,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"764f7ce2_9e94fca7","line":5380,"in_reply_to":"ba5a6629_2844667f","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9c5eb5a6581756d0f7f0b8ab521f3507fe518b1e","unresolved":true,"context_lines":[{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"},{"line_number":5384,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5385,"context_line":"            self.assertEqual(resp.body, b\u0027lalala\u0027)"},{"line_number":5386,"context_line":""},{"line_number":5387,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"dc99ef97_b04fdbcd","line":5384,"updated":"2023-10-04 15:42:25.000000000","message":"I think I\u0027d prefer to use a new Request for each scenario rather than track infocache in my head","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"},{"line_number":5384,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5385,"context_line":"            self.assertEqual(resp.body, b\u0027lalala\u0027)"},{"line_number":5386,"context_line":""},{"line_number":5387,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"0504f7f3_e1d00a0d","line":5384,"in_reply_to":"14e1eb1c_e6e62ee8","updated":"2023-11-02 18:30:49.000000000","message":"Done","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":5381,"context_line":""},{"line_number":5382,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5383,"context_line":"                             slow\u003d[1.0, 1.0])"},{"line_number":5384,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5385,"context_line":"            self.assertEqual(resp.body, b\u0027lalala\u0027)"},{"line_number":5386,"context_line":""},{"line_number":5387,"context_line":"            set_http_connect(200, 200, 200, body\u003db\u0027lalala\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"14e1eb1c_e6e62ee8","line":5384,"in_reply_to":"dc99ef97_b04fdbcd","updated":"2023-10-31 00:31:48.000000000","message":"I think using a clean req for each scenario is totally justifiable; maybe even better as unique test methods (you could always to do a pre-flight request to pre-populate the info cache to get them out of the way).","commit_id":"debf93f775775765b8c86af395379e9f969ab5c2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5376,"context_line":"            self.app.update_request(req)"},{"line_number":5377,"context_line":""},{"line_number":5378,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5379,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5380,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5381,"context_line":"            with self.assertRaises(ChunkReadTimeout):"},{"line_number":5382,"context_line":"                resp.body"}],"source_content_type":"text/x-python","patch_set":4,"id":"5ae4382c_116b309f","side":"PARENT","line":5379,"updated":"2023-10-06 19:27:27.000000000","message":"so it wasn\u0027t obvious, but this was filling in the swift.cache?","commit_id":"d31a54a65c186550a99bb985f056c071690a3939"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":true,"context_lines":[{"line_number":5376,"context_line":"            self.app.update_request(req)"},{"line_number":5377,"context_line":""},{"line_number":5378,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5379,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5380,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5381,"context_line":"            with self.assertRaises(ChunkReadTimeout):"},{"line_number":5382,"context_line":"                resp.body"}],"source_content_type":"text/x-python","patch_set":4,"id":"db35abe5_ec819d50","side":"PARENT","line":5379,"in_reply_to":"5ae4382c_116b309f","updated":"2023-10-09 10:46:58.000000000","message":"The first 200\u0027s are for account and container HEADs, then one obj 200, plus 3 slow reads - the first 2 slow reads are irrelevant (account and container HEADs don\u0027t read)???\n\nThe test passes because the proxy obj controller gets a ChunkReadTimeout followed by 2 StopIterations from the test harness.\n\nBut yes, I guess the cache got filled.","commit_id":"d31a54a65c186550a99bb985f056c071690a3939"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":5376,"context_line":"            self.app.update_request(req)"},{"line_number":5377,"context_line":""},{"line_number":5378,"context_line":"            self.app.recoverable_node_timeout \u003d 0.1"},{"line_number":5379,"context_line":"            set_http_connect(200, 200, 200, slow\u003d[1.0, 1.0, 1.0])"},{"line_number":5380,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5381,"context_line":"            with self.assertRaises(ChunkReadTimeout):"},{"line_number":5382,"context_line":"                resp.body"}],"source_content_type":"text/x-python","patch_set":4,"id":"938f2abd_91831e20","side":"PARENT","line":5379,"in_reply_to":"db35abe5_ec819d50","updated":"2023-10-31 00:31:48.000000000","message":"oh gosh","commit_id":"d31a54a65c186550a99bb985f056c071690a3939"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5345,"context_line":"            self.logger.clear()"},{"line_number":5346,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5347,"context_line":"            self.app.update_request(req)"},{"line_number":5348,"context_line":"            set_http_connect(200, 200, 200, 200, 200,"},{"line_number":5349,"context_line":"                             slow\u003d[0.0, 0.0, 1.0, 1.0, 1.0])"},{"line_number":5350,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5351,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"5da01cfe_ca681a87","line":5348,"updated":"2023-10-06 19:27:27.000000000","message":"the first two 200s are probably for a/c info calls?","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":5345,"context_line":"            self.logger.clear()"},{"line_number":5346,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5347,"context_line":"            self.app.update_request(req)"},{"line_number":5348,"context_line":"            set_http_connect(200, 200, 200, 200, 200,"},{"line_number":5349,"context_line":"                             slow\u003d[0.0, 0.0, 1.0, 1.0, 1.0])"},{"line_number":5350,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5351,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3a38cd29_7485fca1","line":5348,"in_reply_to":"27074bc3_bcbacda1","updated":"2023-11-02 18:30:49.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":5345,"context_line":"            self.logger.clear()"},{"line_number":5346,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5347,"context_line":"            self.app.update_request(req)"},{"line_number":5348,"context_line":"            set_http_connect(200, 200, 200, 200, 200,"},{"line_number":5349,"context_line":"                             slow\u003d[0.0, 0.0, 1.0, 1.0, 1.0])"},{"line_number":5350,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5351,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"27074bc3_bcbacda1","line":5348,"in_reply_to":"5da01cfe_ca681a87","updated":"2023-10-31 00:31:48.000000000","message":"pretty sure that\u0027s definately what\u0027s going on - not sure if there\u0027s an easy way to pre-populate the request\u0027s info cache so it doesn\u0027t need the extra responses for account and container requests.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5355,"context_line":"            for line in error_lines[:3]:"},{"line_number":5356,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5357,"context_line":"            self.assertIn(\u0027Trying to read during GET: ChunkReadTimeout \u0027,"},{"line_number":5358,"context_line":"                          error_lines[3])"},{"line_number":5359,"context_line":""},{"line_number":5360,"context_line":"            # first 2 obj nodes time out"},{"line_number":5361,"context_line":"            self.logger.clear()"}],"source_content_type":"text/x-python","patch_set":4,"id":"9df1ccc2_d9579c8d","line":5358,"updated":"2023-10-06 19:27:27.000000000","message":"i see it now - 4 log lines; one per replicate/backend and one more for client request when the proxy gives up (which happens to include why the last one failed).","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":5355,"context_line":"            for line in error_lines[:3]:"},{"line_number":5356,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5357,"context_line":"            self.assertIn(\u0027Trying to read during GET: ChunkReadTimeout \u0027,"},{"line_number":5358,"context_line":"                          error_lines[3])"},{"line_number":5359,"context_line":""},{"line_number":5360,"context_line":"            # first 2 obj nodes time out"},{"line_number":5361,"context_line":"            self.logger.clear()"}],"source_content_type":"text/x-python","patch_set":4,"id":"a32f3118_0ffe8579","line":5358,"in_reply_to":"9df1ccc2_d9579c8d","updated":"2023-11-02 18:30:49.000000000","message":"except that fourth (new) line was also incrementing the error count on the third node - again! 😞","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5369,"context_line":"            error_lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":5370,"context_line":"            self.assertEqual(2, len(error_lines))"},{"line_number":5371,"context_line":"            for line in error_lines[:2]:"},{"line_number":5372,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5373,"context_line":""},{"line_number":5374,"context_line":"            # now introduce some etags..."},{"line_number":5375,"context_line":"            self.logger.clear()"}],"source_content_type":"text/x-python","patch_set":4,"id":"b9b565d1_e9630947","line":5372,"updated":"2023-10-06 19:27:27.000000000","message":"nice, the loggig asserts help!","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":false,"context_lines":[{"line_number":5369,"context_line":"            error_lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":5370,"context_line":"            self.assertEqual(2, len(error_lines))"},{"line_number":5371,"context_line":"            for line in error_lines[:2]:"},{"line_number":5372,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5373,"context_line":""},{"line_number":5374,"context_line":"            # now introduce some etags..."},{"line_number":5375,"context_line":"            self.logger.clear()"}],"source_content_type":"text/x-python","patch_set":4,"id":"3c63b55e_72fdb4a1","line":5372,"in_reply_to":"b9b565d1_e9630947","updated":"2023-11-02 18:30:49.000000000","message":"Ack","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":true,"context_lines":[{"line_number":5386,"context_line":"            for line in error_lines[:2]:"},{"line_number":5387,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5388,"context_line":""},{"line_number":5389,"context_line":"            # first 2 nodes time out, second has wrong etag"},{"line_number":5390,"context_line":"            self.logger.clear()"},{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"}],"source_content_type":"text/x-python","patch_set":4,"id":"d75c5a50_ae7babee","line":5389,"updated":"2023-10-09 10:46:58.000000000","message":"fix: first 2 nodes would timeout, but second has wrong etag so not used","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":5386,"context_line":"            for line in error_lines[:2]:"},{"line_number":5387,"context_line":"                self.assertIn(\u0027Trying to read object during GET\u0027, line)"},{"line_number":5388,"context_line":""},{"line_number":5389,"context_line":"            # first 2 nodes time out, second has wrong etag"},{"line_number":5390,"context_line":"            self.logger.clear()"},{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"}],"source_content_type":"text/x-python","patch_set":4,"id":"6042ed38_12f9e0e9","line":5389,"in_reply_to":"d75c5a50_ae7babee","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"},{"line_number":5393,"context_line":"            set_http_connect(200, 200, 200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5394,"context_line":"                             slow\u003d[0, 0, 1.0, 1.0],"},{"line_number":5395,"context_line":"                             etags\u003d[\u0027account\u0027, \u0027container\u0027, \u0027a\u0027, \u0027b\u0027, \u0027a\u0027])"},{"line_number":5396,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5397,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa5294a3_c24263de","line":5394,"updated":"2023-10-06 19:27:27.000000000","message":"isn\u0027t the b request slow too?","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2d416742a92cf063f818073040539c379611c666","unresolved":true,"context_lines":[{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"},{"line_number":5393,"context_line":"            set_http_connect(200, 200, 200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5394,"context_line":"                             slow\u003d[0, 0, 1.0, 1.0],"},{"line_number":5395,"context_line":"                             etags\u003d[\u0027account\u0027, \u0027container\u0027, \u0027a\u0027, \u0027b\u0027, \u0027a\u0027])"},{"line_number":5396,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5397,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"40fbb795_1c5929c2","line":5394,"in_reply_to":"36887907_2b2d6bc5","updated":"2023-11-02 18:30:49.000000000","message":"actually, we can truncate the slow list once we\u0027re done with slow nodes, so just [0, 0, 1.0] for account, container, first obj nodes","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":true,"context_lines":[{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"},{"line_number":5393,"context_line":"            set_http_connect(200, 200, 200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5394,"context_line":"                             slow\u003d[0, 0, 1.0, 1.0],"},{"line_number":5395,"context_line":"                             etags\u003d[\u0027account\u0027, \u0027container\u0027, \u0027a\u0027, \u0027b\u0027, \u0027a\u0027])"},{"line_number":5396,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5397,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"36887907_2b2d6bc5","line":5394,"in_reply_to":"e19491b6_2474affc","updated":"2023-10-31 00:31:48.000000000","message":"oh ok, so the slow list doesn\u0027t need an index for etags[3] \u003d \u0027b\u0027 because it\u0027s thrown away w/o ever reading and consuming the slow?  That\u0027s kind of annoying test infra interface.","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3c9741cef3918b6f860cee79ad22d180f908036a","unresolved":true,"context_lines":[{"line_number":5391,"context_line":"            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":5392,"context_line":"            self.app.update_request(req)"},{"line_number":5393,"context_line":"            set_http_connect(200, 200, 200, 200, 200, body\u003db\u0027lalala\u0027,"},{"line_number":5394,"context_line":"                             slow\u003d[0, 0, 1.0, 1.0],"},{"line_number":5395,"context_line":"                             etags\u003d[\u0027account\u0027, \u0027container\u0027, \u0027a\u0027, \u0027b\u0027, \u0027a\u0027])"},{"line_number":5396,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":5397,"context_line":"            self.assertEqual(200, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":4,"id":"e19491b6_2474affc","line":5394,"in_reply_to":"fa5294a3_c24263de","updated":"2023-10-09 10:46:58.000000000","message":"well, no, because it\u0027s not used: the comment \"first 2 nodes time out, second has wrong etag\"is misleading\n\nthe comment at line 5400 is better","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7030789c8861ca2a9d3a4cbc61d85657e8afb9fe","unresolved":true,"context_lines":[{"line_number":11052,"context_line":"            set_http_connect(200, 200, 200, body\u003d\u0027abcdef\u0027, slow\u003d[1.0, 1.0])"},{"line_number":11053,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":11054,"context_line":"            self.assertEqual(200, resp.status_int)"},{"line_number":11055,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":11056,"context_line":"            error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":11057,"context_line":"            self.assertEqual(1, len(error_lines))"},{"line_number":11058,"context_line":"            self.assertIn(\u0027Trying to read during GET\u0027, error_lines[0])"}],"source_content_type":"text/x-python","patch_set":4,"id":"a0940c44_4991b350","line":11055,"updated":"2023-10-06 19:27:27.000000000","message":"this actually looks a lot more reasonable that test asserting the ChunkReadTimeout leaked out to WSGI?","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":11052,"context_line":"            set_http_connect(200, 200, 200, body\u003d\u0027abcdef\u0027, slow\u003d[1.0, 1.0])"},{"line_number":11053,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":11054,"context_line":"            self.assertEqual(200, resp.status_int)"},{"line_number":11055,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":11056,"context_line":"            error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":11057,"context_line":"            self.assertEqual(1, len(error_lines))"},{"line_number":11058,"context_line":"            self.assertIn(\u0027Trying to read during GET\u0027, error_lines[0])"}],"source_content_type":"text/x-python","patch_set":4,"id":"9ed8d73d_2fd182df","line":11055,"in_reply_to":"a0940c44_4991b350","updated":"2023-10-31 00:31:48.000000000","message":"Done","commit_id":"38b9534de2738ec94b686f974db2e4d24efc2463"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c4f9242d005b054cd491fe1e60430823ecc5265c","unresolved":false,"context_lines":[{"line_number":11059,"context_line":"                resp.body"},{"line_number":11060,"context_line":"            error_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":11061,"context_line":"            self.assertEqual(1, len(error_lines))"},{"line_number":11062,"context_line":"            self.assertIn(\u0027Trying to read during GET\u0027, error_lines[0])"},{"line_number":11063,"context_line":""},{"line_number":11064,"context_line":""},{"line_number":11065,"context_line":"@patch_policies([StoragePolicy(0, \u0027zero\u0027, True, object_ring\u003dFakeRing())])"}],"source_content_type":"text/x-python","patch_set":7,"id":"0b7ad380_98b809ba","line":11062,"updated":"2023-10-31 00:31:48.000000000","message":"seems like this is the new error log line in the !\u003d Object branch","commit_id":"077cc62b82ba97dbaf4f742db6050fd3b08a047c"}]}