)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"460598e416be769e5f4e6555007e689cf30e9dd1","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Clay Gerrard \u003cclay.gerrard@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-06-22 17:33:38 -0500"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add concurrent_gets to EC GET requests"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Change-Id: Ia0a9398107a400815be2e0097b1b8e76336a0253"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"bf51134e_e7a369ff","line":7,"updated":"2020-08-11 06:32:46.000000000","message":"note the most descriptive commit message, but I realise this is a WIP and may be subject to change.. just couldn\u0027t help myself","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"}],"etc/proxy-server.conf-sample":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"56347d38a8d5ba4beea6f6e578de56723abe27e1","unresolved":false,"context_lines":[{"line_number":202,"context_line":"#"},{"line_number":203,"context_line":"# By default on a GET/HEAD swift will connect to a minimum number storage nodes"},{"line_number":204,"context_line":"# in a minimum number of threads - for replicated data just a single request to"},{"line_number":205,"context_line":"# a single node one at a time.  When enabled concurrent_gets allows the proxy,"},{"line_number":206,"context_line":"# to use up to replica count threads when waiting on a response.  In"},{"line_number":207,"context_line":"# conjunction with the concurrency_timeout option this will allow swift to send"},{"line_number":208,"context_line":"# out GET/HEAD requests to the storage nodes concurrently and answer as soon as"},{"line_number":209,"context_line":"# the minimum number of backend responses are availabe - in replicated contexts"}],"source_content_type":"application/octet-stream","patch_set":20,"id":"9f560f44_819c356b","line":206,"range":{"start_line":205,"start_character":32,"end_line":206,"end_character":64},"updated":"2020-08-31 21:39:52.000000000","message":"nit: \"When enabled, concurrent_gets allows the proxy to use up to replica count threads when waiting on a response.\" (misplaced comma)","commit_id":"8f60e0a2607514f05fb873e4a313ab4a93df7601"}],"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":3374,"context_line":""},{"line_number":3375,"context_line":"    @property"},{"line_number":3376,"context_line":"    def ready(self):"},{"line_number":3377,"context_line":"        return max(0, self._pending - self._inflight)"},{"line_number":3378,"context_line":""},{"line_number":3379,"context_line":"    def spawn(self, func, *args, **kwargs):"},{"line_number":3380,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_205c434a","line":3377,"updated":"2020-06-17 13:13:52.000000000","message":"i might be able to pull this out, although I do use it sometimes in debugging; maybe just add a doc string","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":3374,"context_line":""},{"line_number":3375,"context_line":"    @property"},{"line_number":3376,"context_line":"    def ready(self):"},{"line_number":3377,"context_line":"        return max(0, self._pending - self._inflight)"},{"line_number":3378,"context_line":""},{"line_number":3379,"context_line":"    def spawn(self, func, *args, **kwargs):"},{"line_number":3380,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_003f6e25","line":3377,"in_reply_to":"bf51134e_205c434a","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c45cfaea5156354c87102214aff01bb806019f6e","unresolved":false,"context_lines":[{"line_number":858,"context_line":"class GetOrHeadHandler(object):"},{"line_number":859,"context_line":"    def __init__(self, app, req, server_type, node_iter, partition, path,"},{"line_number":860,"context_line":"                 backend_headers, concurrency\u003d1, client_chunk_size\u003dNone,"},{"line_number":861,"context_line":"                 newest\u003dNone, header_provider\u003dNone):"},{"line_number":862,"context_line":"        self.app \u003d app"},{"line_number":863,"context_line":"        self.node_iter \u003d node_iter"},{"line_number":864,"context_line":"        self.server_type \u003d server_type"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_3efa51bc","line":861,"updated":"2020-05-15 15:10:42.000000000","message":"there\u0027s so many little threads like \"header_provider\" that were entirely EC specific and will be able to be pulled out of the replicated GET handling.","commit_id":"4b682eef13454842bf5e0fc4460c72e7d0ec11fa"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":1412,"context_line":"        return None, None"},{"line_number":1413,"context_line":""},{"line_number":1414,"context_line":""},{"line_number":1415,"context_line":"class GetOrHeadHandler(ResumingGetter):"},{"line_number":1416,"context_line":"    def _make_app_iter(self, req, node, source):"},{"line_number":1417,"context_line":"        \"\"\""},{"line_number":1418,"context_line":"        Returns an iterator over the contents of the source (via its read"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_d2003d05","side":"PARENT","line":1415,"updated":"2020-06-02 17:27:36.000000000","message":"So this is essentially the first step towards a revert/redo of https://review.opendev.org/#/c/173497/\n\nA lot has happened in 5 years, it\u0027s not obvious we made any mistakes then and no guarantees our path from here to the next most obviously better thing we can do is the straightest possible path to get to where we want to be 5 more years from now... but I *do* think some renovation here is a good technical investment.","commit_id":"4f440ad44d16fcc51dd060ca7040e3ed5c7b7483"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":1547,"context_line":""},{"line_number":1548,"context_line":"    def _node_gen(self):"},{"line_number":1549,"context_line":"        while self.primary_nodes:"},{"line_number":1550,"context_line":"            node \u003d self.primary_nodes.pop(0)"},{"line_number":1551,"context_line":"            if not self.app.error_limited(node):"},{"line_number":1552,"context_line":"                yield node"},{"line_number":1553,"context_line":"                if not self.app.error_limited(node):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_a035f3f9","line":1550,"updated":"2020-06-17 13:13:52.000000000","message":"this mutation is related to the num_primary_nodes change - I could possibly test this independently.","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":1547,"context_line":""},{"line_number":1548,"context_line":"    def _node_gen(self):"},{"line_number":1549,"context_line":"        while self.primary_nodes:"},{"line_number":1550,"context_line":"            node \u003d self.primary_nodes.pop(0)"},{"line_number":1551,"context_line":"            if not self.app.error_limited(node):"},{"line_number":1552,"context_line":"                yield node"},{"line_number":1553,"context_line":"                if not self.app.error_limited(node):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_227a56f0","line":1550,"in_reply_to":"bf51134e_a035f3f9","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1011,"context_line":"            return True"},{"line_number":1012,"context_line":"        return is_success(src.status) or is_redirection(src.status)"},{"line_number":1013,"context_line":""},{"line_number":1014,"context_line":"    def response_parts_iter(self, req):"},{"line_number":1015,"context_line":"        source, node \u003d self._get_source_and_node()"},{"line_number":1016,"context_line":"        it \u003d None"},{"line_number":1017,"context_line":"        if source:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_79262a6a","side":"PARENT","line":1014,"updated":"2020-06-29 20:27:20.000000000","message":"In retrospect, this was such a code-smell -- having a generic-looking hook in base.py that\u0027s only actually used in *one place* buried down in the EC half of the object controller? Definitely an argument in favor of the code split in the parent patch.","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1500,"context_line":""},{"line_number":1501,"context_line":"        # Use of list() here forcibly yanks the first N nodes (the primary"},{"line_number":1502,"context_line":"        # nodes) from node_iter, so the rest of its values are handoffs."},{"line_number":1503,"context_line":"        self.primary_nodes \u003d self.app.sort_nodes("},{"line_number":1504,"context_line":"            list(itertools.islice(node_iter, self.num_primary_nodes)),"},{"line_number":1505,"context_line":"            policy\u003dpolicy)"},{"line_number":1506,"context_line":"        self.handoff_iter \u003d node_iter"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_19d8ce50","line":1503,"updated":"2020-06-29 20:27:20.000000000","message":"OK, so previously we had this static list, now we\u0027ll pop off nodes as we make requests.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1508,"context_line":""},{"line_number":1509,"context_line":"    @property"},{"line_number":1510,"context_line":"    def primaries_left(self):"},{"line_number":1511,"context_line":"        return len(self.primary_nodes)"},{"line_number":1512,"context_line":""},{"line_number":1513,"context_line":"    def __iter__(self):"},{"line_number":1514,"context_line":"        self._node_iter \u003d self._node_gen()"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_55e79b47","line":1511,"updated":"2020-06-30 14:43:40.000000000","message":"the mutably list is entirely in service of answering the question of \"is there any more primaries left\" since that\u0027s when the feeder shuts down","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1549,"context_line":"        self._node_provider \u003d callback"},{"line_number":1550,"context_line":""},{"line_number":1551,"context_line":"    def _node_gen(self):"},{"line_number":1552,"context_line":"        while self.primary_nodes:"},{"line_number":1553,"context_line":"            node \u003d self.primary_nodes.pop(0)"},{"line_number":1554,"context_line":"            if not self.app.error_limited(node):"},{"line_number":1555,"context_line":"                yield node"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f9c7da62","line":1552,"updated":"2020-06-29 20:27:20.000000000","message":"Is this intended as a straight translation of the old behavior, or are we looking for the ability to add primaries back to the self.primary_nodes list?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":1292,"context_line":"            else:"},{"line_number":1293,"context_line":"                if self.used_source_etag and \\"},{"line_number":1294,"context_line":"                    self.used_source_etag !\u003d normalize_etag(src_headers.get("},{"line_number":1295,"context_line":"                        \u0027x-object-sysmeta-ec-etag\u0027,"},{"line_number":1296,"context_line":"                        src_headers.get(\u0027etag\u0027, \u0027\u0027))):"},{"line_number":1297,"context_line":"                    self.statuses.append(HTTP_NOT_FOUND)"},{"line_number":1298,"context_line":"                    self.reasons.append(\u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_7170c573","line":1295,"range":{"start_line":1295,"start_character":24,"end_line":1295,"end_character":50},"updated":"2020-07-29 01:42:38.000000000","message":"Hmm...\n\nWe shouldn\u0027t ever encounter this header anymore, right?","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":1398,"context_line":"            # versions of the same object, we might end up switching between"},{"line_number":1399,"context_line":"            # old and new mid-stream and giving garbage to the client."},{"line_number":1400,"context_line":"            self.used_source_etag \u003d normalize_etag(src_headers.get("},{"line_number":1401,"context_line":"                \u0027x-object-sysmeta-ec-etag\u0027, src_headers.get(\u0027etag\u0027, \u0027\u0027)))"},{"line_number":1402,"context_line":"            self.node \u003d node"},{"line_number":1403,"context_line":"            return source, node"},{"line_number":1404,"context_line":"        return None, None"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_d16d5199","line":1401,"range":{"start_line":1401,"start_character":16,"end_line":1401,"end_character":42},"updated":"2020-07-29 01:42:38.000000000","message":"Here, too.","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":1544,"context_line":""},{"line_number":1545,"context_line":"    def _node_gen(self):"},{"line_number":1546,"context_line":"        while self.primary_nodes:"},{"line_number":1547,"context_line":"            node \u003d self.primary_nodes.pop(0)"},{"line_number":1548,"context_line":"            if not self.app.error_limited(node):"},{"line_number":1549,"context_line":"                yield node"},{"line_number":1550,"context_line":"                if not self.app.error_limited(node):"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_4f87b110","line":1547,"updated":"2020-08-19 05:39:21.000000000","message":"(This is O(n**2), but it shouldn\u0027t matter for the small numbers of nodes we\u0027d have here.)","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c45cfaea5156354c87102214aff01bb806019f6e","unresolved":false,"context_lines":[{"line_number":2763,"context_line":"        it \u003d None"},{"line_number":2764,"context_line":"        if source:"},{"line_number":2765,"context_line":"            it \u003d self._get_response_parts_iter(req, node, source)"},{"line_number":2766,"context_line":"        return it"},{"line_number":2767,"context_line":""},{"line_number":2768,"context_line":""},{"line_number":2769,"context_line":"@ObjectControllerRouter.register(EC_POLICY)"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_9e66fdc0","line":2766,"updated":"2020-05-15 15:10:42.000000000","message":"There\u0027s still a lot of duplication here with base for the multi-range handling; some of that might not go away - but I\u0027m hoping after I simplify and separate that as much I can in the focused domain of EC GETs it\u0027ll be more obvious how much if any we can put back together with base","commit_id":"4b682eef13454842bf5e0fc4460c72e7d0ec11fa"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c45cfaea5156354c87102214aff01bb806019f6e","unresolved":false,"context_lines":[{"line_number":2888,"context_line":"                    timeout \u003d max(0, batch_deadline - time.time())"},{"line_number":2889,"context_line":"                    if timeout \u003d\u003d 0.0:"},{"line_number":2890,"context_line":"                        # not so fast there cheif"},{"line_number":2891,"context_line":"                        sleep()"},{"line_number":2892,"context_line":"                else:"},{"line_number":2893,"context_line":"                    timeout \u003d None"},{"line_number":2894,"context_line":"                ready_resp \u003d pile.waitfirst(timeout)"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_9e0b9dd4","line":2891,"updated":"2020-05-15 15:10:42.000000000","message":"This section of code is what I really want to be working on, it can be made simpler.\n\nIt\u0027s already very much dependent on ECFragGetters *not* making additional requests after errors or timeouts and it will continue to drive more changes to that interface that move it further away from the replicated/base controllers GET handling.","commit_id":"4b682eef13454842bf5e0fc4460c72e7d0ec11fa"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":2254,"context_line":"            return nodes.pop(0).copy()"},{"line_number":2255,"context_line":""},{"line_number":2256,"context_line":""},{"line_number":2257,"context_line":"class ECFragGetter(object):"},{"line_number":2258,"context_line":""},{"line_number":2259,"context_line":"    def __init__(self, app, req, node_iter, partition, policy, path,"},{"line_number":2260,"context_line":"                 backend_headers, header_provider\u003dNone):"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_729b3116","line":2257,"updated":"2020-06-02 17:27:36.000000000","message":"This is *obviously* a riff on GetOrHeadHandler/ResumingGetter\n\nhttps://github.com/openstack/swift/blob/master/swift/proxy/controllers/base.py#L1415\n\nFor now I believe the most important work requires us to be able to change this interface *freely* without having to serve any other masters besides the ECObjectController - it\u0027s already starting to diverge in subtle/non-obvious ways.\n\nThe differences will have to be accounted for at some point, probably follow-on patch that tries to \"pull out\" all of the EC specific stuff from GetOrHeadHandler that is no longer used (hopefully that will be a nearly negative diff except maybe additional tests)\n\nAnd then *finally* we might get to see what code (if any) they really *should* (could?) share and a reasonable way to build them on a reasonable foundation.  But if the final state is two \"more easy to change and reason about\" concrete classes hand-crafted from just an object that do similar things but don\u0027t share code...  I can probably still maintain that and live with myself.","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":2553,"context_line":"                break"},{"line_number":2554,"context_line":"            if source and self.is_good_source(source):"},{"line_number":2555,"context_line":"                return source, node"},{"line_number":2556,"context_line":"        return None, None"},{"line_number":2557,"context_line":""},{"line_number":2558,"context_line":"    def _get_response_parts_iter(self, req, node, source):"},{"line_number":2559,"context_line":"        # Someday we can replace this [mess] with python 3\u0027s \"nonlocal\""}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_d4530a15","line":2556,"updated":"2020-06-02 17:27:36.000000000","message":"This is a new internal \"interface\" that only ECFragGetter has\n\nThe difference between \"_get_source_and_node\" (exposed via response_parts_iter) which can fail and return None, and \"_dig_for_source_and_node\" which will iterate through handoffs and is only consumed internally *after* we\u0027ve already been selected is probably important/significant... somehow?","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":2958,"context_line":"                        extra_requests +\u003d 1"},{"line_number":2959,"context_line":"                        pile.spawn(self._fragment_GET_request,"},{"line_number":2960,"context_line":"                                   req, safe_iter, partition,"},{"line_number":2961,"context_line":"                                   policy, buckets.get_extra_headers)"},{"line_number":2962,"context_line":""},{"line_number":2963,"context_line":"        # Put this back, since we *may* need it for kickoff()/_fix_response()"},{"line_number":2964,"context_line":"        # (but note that _fix_ranges() may also pop it back off before then)"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_b47136ab","line":2961,"updated":"2020-06-02 17:27:36.000000000","message":"MOST of the work we want to concern ourselves with when thinking about \"waterfall ec\" is going to happen in these 60 or so lines; this is *definitely* not the final state; but it already is yielding some new positive behaviors which I plan to expand on","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2759,"context_line":"            _timeout \u003d self.app.concurrency_timeout \\"},{"line_number":2760,"context_line":"                if pile.inflight \u003c self.concurrency else None"},{"line_number":2761,"context_line":"            if pile.waitfirst(_timeout):"},{"line_number":2762,"context_line":"                break"},{"line_number":2763,"context_line":"        else:"},{"line_number":2764,"context_line":"            # ran out of nodes, see if any stragglers will finish"},{"line_number":2765,"context_line":"            any(pile)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_062dd894","side":"PARENT","line":2762,"updated":"2020-06-17 13:13:52.000000000","message":"related to the refactoring and code duplication - THIS code that had to be pulled out of the middle of this method - IS replicated concurrent gets; the heart of it anyway.","commit_id":"ffda7b149fb53ce88b6c92aad8f66b558967fbc6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2762,"context_line":"                break"},{"line_number":2763,"context_line":"        else:"},{"line_number":2764,"context_line":"            # ran out of nodes, see if any stragglers will finish"},{"line_number":2765,"context_line":"            any(pile)"},{"line_number":2766,"context_line":""},{"line_number":2767,"context_line":"        # this helps weed out any sucess status that were found before a 404"},{"line_number":2768,"context_line":"        # and added to the list in the case of x-newest."}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_2628dc84","side":"PARENT","line":2765,"updated":"2020-06-17 13:13:52.000000000","message":"below in waterfall-ec this is implicit with the \"for resp in pile\" phrasing - even if we run out nodes (which some tests do)","commit_id":"ffda7b149fb53ce88b6c92aad8f66b558967fbc6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2943,"context_line":"                    shortfall \u003d best_bucket.shortfall"},{"line_number":2944,"context_line":"                    if not best_bucket.durable and shortfall \u003c\u003d 0:"},{"line_number":2945,"context_line":"                        # be willing to go a *little* deeper, slowly"},{"line_number":2946,"context_line":"                        shortfall \u003d 1"},{"line_number":2947,"context_line":"                    shortfall \u003d min(shortfall, bad_bucket.shortfall)"},{"line_number":2948,"context_line":"                if (extra_requests \u003c max_extra_requests and"},{"line_number":2949,"context_line":"                        shortfall \u003e pile._pending and"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_c932d3f1","side":"PARENT","line":2946,"updated":"2020-06-17 13:13:52.000000000","message":"I remember cursing this kludge went in: \"worst case it\u0027s better to be slow and correct than fast and wrong; if it\u0027s a problem we\u0027ll fix it later\" kinda sentiment.  The behavior is critical to supporting the guarantees the proxy has for returning durable data.\n\nI should add logging here and see if it correlates at all with a small percentage of outlying slow EC responses.","commit_id":"ffda7b149fb53ce88b6c92aad8f66b558967fbc6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":1480,"context_line":"        None if client_end is None else"},{"line_number":1481,"context_line":"        # bytes M-N"},{"line_number":1482,"context_line":"        (((int(client_end // segment_size) + 1) *"},{"line_number":1483,"context_line":"          segment_size) - 1) if client_start is not None else"},{"line_number":1484,"context_line":"        # bytes -N: we get some extra bytes to make sure we"},{"line_number":1485,"context_line":"        # have all we need."},{"line_number":1486,"context_line":"        #"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_22c89620","line":1483,"updated":"2020-06-18 21:58:08.000000000","message":"none of these need to be in this diff - distracting #willfix","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2102,"context_line":"        \"\"\""},{"line_number":2103,"context_line":"        self.policy \u003d policy"},{"line_number":2104,"context_line":"        self.buckets \u003d {}"},{"line_number":2105,"context_line":"        self.bad_buckets \u003d {None: ECGetResponseBucket(self.policy, None)}"},{"line_number":2106,"context_line":"        self.node_iter_count \u003d 0"},{"line_number":2107,"context_line":""},{"line_number":2108,"context_line":"    def _get_bucket(self, timestamp_str):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_29370fb3","line":2105,"updated":"2020-06-17 13:13:52.000000000","message":"I\u0027m *pretty* sure having the collection track bad responses by status code like it tracks good responses by timestamp is a good idea.  almost positive.","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e18e1c8845f1316dc1bc7825a0427d71adddd85a","unresolved":false,"context_lines":[{"line_number":2131,"context_line":"        :raises ValueError: if the response etag or status code values do not"},{"line_number":2132,"context_line":"            match any values previously received for the same timestamp"},{"line_number":2133,"context_line":"        \"\"\""},{"line_number":2134,"context_line":"        if is_success(get.last_status):"},{"line_number":2135,"context_line":"            self.add_good_response(get, parts_iter)"},{"line_number":2136,"context_line":"        else:"},{"line_number":2137,"context_line":"            self.add_bad_resp(get, parts_iter)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_499fd15e","line":2134,"range":{"start_line":2134,"start_character":26,"end_line":2134,"end_character":37},"updated":"2020-06-17 18:07:28.000000000","message":"This could be None, leading to (most of?) the py3 unit test failures. Probably want\n\n if get.last_status and is_success(get.last_status):\n\n? I think I\u0027m OK with the TypeError passing None to is_success...","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2190,"context_line":""},{"line_number":2191,"context_line":"        The \"best\" bucket is the newest timestamp with sufficient getters, or"},{"line_number":2192,"context_line":"        the closest to having sufficient getters, unless it is bettered by a"},{"line_number":2193,"context_line":"        bucket with potential alternate nodes."},{"line_number":2194,"context_line":""},{"line_number":2195,"context_line":"        :return: An instance of :class:`~ECGetResponseBucket` or None if there"},{"line_number":2196,"context_line":"                 are no buckets in the collection."}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_093ccb91","line":2193,"updated":"2020-06-17 13:13:52.000000000","message":"doc string needs an update","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2190,"context_line":""},{"line_number":2191,"context_line":"        The \"best\" bucket is the newest timestamp with sufficient getters, or"},{"line_number":2192,"context_line":"        the closest to having sufficient getters, unless it is bettered by a"},{"line_number":2193,"context_line":"        bucket with potential alternate nodes."},{"line_number":2194,"context_line":""},{"line_number":2195,"context_line":"        :return: An instance of :class:`~ECGetResponseBucket` or None if there"},{"line_number":2196,"context_line":"                 are no buckets in the collection."}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_22f1b66c","line":2193,"in_reply_to":"bf51134e_093ccb91","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2304,"context_line":"    \"\"\""},{"line_number":2305,"context_line":"    if status \u003d\u003d 416:"},{"line_number":2306,"context_line":"        return True"},{"line_number":2307,"context_line":"    return is_success(status) or is_redirection(status)"},{"line_number":2308,"context_line":""},{"line_number":2309,"context_line":""},{"line_number":2310,"context_line":"class ECFragGetter(object):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_a94aff23","line":2307,"updated":"2020-06-17 13:13:52.000000000","message":"I should probably maybe this back to base","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2304,"context_line":"    \"\"\""},{"line_number":2305,"context_line":"    if status \u003d\u003d 416:"},{"line_number":2306,"context_line":"        return True"},{"line_number":2307,"context_line":"    return is_success(status) or is_redirection(status)"},{"line_number":2308,"context_line":""},{"line_number":2309,"context_line":""},{"line_number":2310,"context_line":"class ECFragGetter(object):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_02c2b2e9","line":2307,"in_reply_to":"bf51134e_a94aff23","updated":"2020-06-18 21:58:08.000000000","message":"bah, nope!  base has a if self.server_type \u003d\u003d \u0027Object\u0027 check","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2668,"context_line":"        if self.statuses:"},{"line_number":2669,"context_line":"            return self.statuses[-1]"},{"line_number":2670,"context_line":"        else:"},{"line_number":2671,"context_line":"            return None"},{"line_number":2672,"context_line":""},{"line_number":2673,"context_line":"    @property"},{"line_number":2674,"context_line":"    def last_headers(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_464b5083","line":2671,"updated":"2020-06-17 13:13:52.000000000","message":"py3 thinks this is an interesting stub :)","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2668,"context_line":"        if self.statuses:"},{"line_number":2669,"context_line":"            return self.statuses[-1]"},{"line_number":2670,"context_line":"        else:"},{"line_number":2671,"context_line":"            return None"},{"line_number":2672,"context_line":""},{"line_number":2673,"context_line":"    @property"},{"line_number":2674,"context_line":"    def last_headers(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_42bcaa68","line":2671,"in_reply_to":"bf51134e_464b5083","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2787,"context_line":"        # statuses to list of all responses\" interface"},{"line_number":2788,"context_line":"        node \u003d next(self.node_iter)"},{"line_number":2789,"context_line":"        self._make_node_request(node, node_timeout,"},{"line_number":2790,"context_line":"                                self.app.logger.thread_locals)"},{"line_number":2791,"context_line":""},{"line_number":2792,"context_line":"        # this helps weed out any sucess status that were found before a 404"},{"line_number":2793,"context_line":"        # and added to the list in the case of x-newest."}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_460eb034","line":2790,"updated":"2020-06-17 13:13:52.000000000","message":"I\u0027m sure there\u0027s a way to put this back together rather than pull it further apart, something about _get_s\u0026n vs _dig_for_s\u0026n\n\nI\u0027m still open to what direction we might go on-top of this.","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2898,"context_line":"                                 buckets, feeder_q):"},{"line_number":2899,"context_line":"        while True:"},{"line_number":2900,"context_line":"            try:"},{"line_number":2901,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2902,"context_line":"            except Empty:"},{"line_number":2903,"context_line":"                if safe_iter.unsafe_iter.primary_nodes:"},{"line_number":2904,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_c632c0f1","line":2901,"updated":"2020-06-17 13:13:52.000000000","message":"in a follow on patch, this will be a per-policy setting\n\nI\u0027d also like to be able to express other step-wise timeout functions that Alex and Tim have described using some sort of configuration DSL","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2900,"context_line":"            try:"},{"line_number":2901,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2902,"context_line":"            except Empty:"},{"line_number":2903,"context_line":"                if safe_iter.unsafe_iter.primary_nodes:"},{"line_number":2904,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2905,"context_line":"                               req, safe_iter, partition,"},{"line_number":2906,"context_line":"                               policy, buckets.get_extra_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_a6406c37","line":2903,"updated":"2020-06-17 13:13:52.000000000","message":"this feels a bit behind the curtins - like neither unsafe_iter nor the primary nodes (mutating list!) are private (but they probably reasonably could be!)","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2900,"context_line":"            try:"},{"line_number":2901,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2902,"context_line":"            except Empty:"},{"line_number":2903,"context_line":"                if safe_iter.unsafe_iter.primary_nodes:"},{"line_number":2904,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2905,"context_line":"                               req, safe_iter, partition,"},{"line_number":2906,"context_line":"                               policy, buckets.get_extra_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_626e0e02","line":2903,"in_reply_to":"bf51134e_a6406c37","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2972,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("},{"line_number":2973,"context_line":"                    node_iter.nodes_left \u003e 0 or buckets.has_alternate_node())"},{"line_number":2974,"context_line":"                good_resp_not_durable \u003d (best_bucket.timestamp and not"},{"line_number":2975,"context_line":"                                         best_bucket.durable)"},{"line_number":2976,"context_line":"                bad_resp \u003d not is_good_source(get.last_status)"},{"line_number":2977,"context_line":"                if requests_available and ("},{"line_number":2978,"context_line":"                        buckets.shortfall \u003e pile._pending or"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_c9e3f332","line":2975,"updated":"2020-06-17 13:13:52.000000000","message":"this is more aggressive than the old kludge which would wait until we had a full ndata set of non-durable frags before spawning a single additional request at a time\n\nHere we\u0027re keeping all ndata \"searchers\" poking around nodes- maybe even nreplicas if the primary feeder has fired - all while holding onto upto ndata-1 non-durable responses.\n\na middle ground might look like len(best_bucket.gets) \u003e nparity or slide back towards the shortfall \u003e _pending test on the less aggressive end\n\nthere\u0027s a lot of different ways this collection of bools could be re-grouped; maybe durability *should* be proportionally related to shortfall.","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2976,"context_line":"                bad_resp \u003d not is_good_source(get.last_status)"},{"line_number":2977,"context_line":"                if requests_available and ("},{"line_number":2978,"context_line":"                        buckets.shortfall \u003e pile._pending or"},{"line_number":2979,"context_line":"                        good_resp_not_durable or bad_resp):"},{"line_number":2980,"context_line":"                    extra_requests +\u003d 1"},{"line_number":2981,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2982,"context_line":"                               req, safe_iter, partition,"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_a624ecc7","line":2979,"updated":"2020-06-17 13:13:52.000000000","message":"I\u0027ve started to think of this as - in all but the most unexceptional case each of our ndata \"lanes\" start another request to make up for the one that just \"failed\" (or in the case of multi-buckets: didn\u0027t decrease our shortfall proportionally to _pending)","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2056,"context_line":"    @property"},{"line_number":2057,"context_line":"    def shortfall(self):"},{"line_number":2058,"context_line":"        result \u003d self.policy.ec_ndata - len(self.get_responses())"},{"line_number":2059,"context_line":"        return max(result, 0)"},{"line_number":2060,"context_line":""},{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall_with_alts(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_99ae9e2b","side":"PARENT","line":2059,"range":{"start_line":2059,"start_character":27,"end_line":2059,"end_character":28},"updated":"2020-06-29 20:27:20.000000000","message":"I need to refresh my memory about why https://github.com/openstack/swift/commit/3189410f changed this from\n\n 0 if self._durable else 1","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2056,"context_line":"    @property"},{"line_number":2057,"context_line":"    def shortfall(self):"},{"line_number":2058,"context_line":"        result \u003d self.policy.ec_ndata - len(self.get_responses())"},{"line_number":2059,"context_line":"        return max(result, 0)"},{"line_number":2060,"context_line":""},{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall_with_alts(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_5535dba7","side":"PARENT","line":2059,"range":{"start_line":2059,"start_character":27,"end_line":2059,"end_character":28},"in_reply_to":"bf51134e_99ae9e2b","updated":"2020-06-30 14:43:40.000000000","message":"the long winded explanation is probably https://etherpad.opendev.org/p/the-non-durable-problem\n\nwhile (ab?)using shortfall is maybe expedient; I\u0027ll agree it\u0027s not obvious (or wasn\u0027t so to me)","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2261,"context_line":"        self.req_path \u003d req.path"},{"line_number":2262,"context_line":"        self.req_query_string \u003d req.query_string"},{"line_number":2263,"context_line":"        if newest is None:"},{"line_number":2264,"context_line":"            self.newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, \u0027f\u0027))"},{"line_number":2265,"context_line":"        else:"},{"line_number":2266,"context_line":"            self.newest \u003d newest"},{"line_number":2267,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_14bd33b6","side":"PARENT","line":2264,"updated":"2020-06-29 20:27:20.000000000","message":"Oh, interesting -- so EC is losing X-Newest support...\n\nGood riddance, I suppose.","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2261,"context_line":"        self.req_path \u003d req.path"},{"line_number":2262,"context_line":"        self.req_query_string \u003d req.query_string"},{"line_number":2263,"context_line":"        if newest is None:"},{"line_number":2264,"context_line":"            self.newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, \u0027f\u0027))"},{"line_number":2265,"context_line":"        else:"},{"line_number":2266,"context_line":"            self.newest \u003d newest"},{"line_number":2267,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_35d66729","side":"PARENT","line":2264,"in_reply_to":"bf51134e_14bd33b6","updated":"2020-06-30 14:43:40.000000000","message":"i don\u0027t think x-newest really... \"worked\" in that we\u0027re not often going to find two rebuild-able versions of the same object - but if we did we already/always returned the newest.","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2389,"context_line":"        :param src: the response from the backend"},{"line_number":2390,"context_line":"        :returns: True if found, False if not"},{"line_number":2391,"context_line":"        \"\"\""},{"line_number":2392,"context_line":"        if self.server_type \u003d\u003d \u0027Object\u0027 and src.status \u003d\u003d 416:"},{"line_number":2393,"context_line":"            return True"},{"line_number":2394,"context_line":"        return is_success(src.status) or is_redirection(src.status)"},{"line_number":2395,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d4fbbbba","side":"PARENT","line":2392,"updated":"2020-06-29 20:27:20.000000000","message":"Ah, OK -- I\u0027m starting to see the genesis of the 416 carve-out...","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2644,"context_line":"            return False"},{"line_number":2645,"context_line":"        req_headers \u003d dict(self.backend_headers)"},{"line_number":2646,"context_line":"        # a request may be specialised with specific backend headers"},{"line_number":2647,"context_line":"        if self.header_provider:"},{"line_number":2648,"context_line":"            req_headers.update(self.header_provider())"},{"line_number":2649,"context_line":"        start_node_timing \u003d time.time()"},{"line_number":2650,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_94914355","side":"PARENT","line":2647,"range":{"start_line":2647,"start_character":16,"end_line":2647,"end_character":31},"updated":"2020-06-29 20:27:20.000000000","message":"Can we drop header_provider support from base.py now?","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2644,"context_line":"            return False"},{"line_number":2645,"context_line":"        req_headers \u003d dict(self.backend_headers)"},{"line_number":2646,"context_line":"        # a request may be specialised with specific backend headers"},{"line_number":2647,"context_line":"        if self.header_provider:"},{"line_number":2648,"context_line":"            req_headers.update(self.header_provider())"},{"line_number":2649,"context_line":"        start_node_timing \u003d time.time()"},{"line_number":2650,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d5cd6b87","side":"PARENT","line":2647,"range":{"start_line":2647,"start_character":16,"end_line":2647,"end_character":31},"in_reply_to":"bf51134e_94914355","updated":"2020-06-30 14:43:40.000000000","message":"yeah if we want to go the fully divergent route there\u0027s probably a number of shims cut through the replicated handling that were only ever used for ec","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2807,"context_line":"                              partition, req.swift_entity_path,"},{"line_number":2808,"context_line":"                              backend_headers,"},{"line_number":2809,"context_line":"                              client_chunk_size\u003dpolicy.fragment_size,"},{"line_number":2810,"context_line":"                              newest\u003dFalse, header_provider\u003dheader_provider)"},{"line_number":2811,"context_line":"        return (getter, getter.response_parts_iter(req))"},{"line_number":2812,"context_line":""},{"line_number":2813,"context_line":"    def _convert_range(self, req, policy):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_1419b3d5","side":"PARENT","line":2810,"range":{"start_line":2810,"start_character":30,"end_line":2810,"end_character":42},"updated":"2020-06-29 20:27:20.000000000","message":"Ah -- EC *never had* X-Newest support. Got it.","commit_id":"2aa4fe6c6a23c7b1eafe211dcb7ba779a6f358b9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2005,"context_line":"    def add_response(self, getter, parts_iter):"},{"line_number":2006,"context_line":"        headers \u003d getter.last_headers"},{"line_number":2007,"context_line":"        timestamp_str \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2008,"context_line":"                                    headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2009,"context_line":"        if timestamp_str and Timestamp(timestamp_str) \u003e self.timestamp:"},{"line_number":2010,"context_line":"            self.timestamp_str \u003d timestamp_str"},{"line_number":2011,"context_line":"        if not self.gets:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f99cfa6a","line":2008,"updated":"2020-06-29 20:27:20.000000000","message":"I always need a refresher: is this going to get us the timestamp for .data or .meta?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2005,"context_line":"    def add_response(self, getter, parts_iter):"},{"line_number":2006,"context_line":"        headers \u003d getter.last_headers"},{"line_number":2007,"context_line":"        timestamp_str \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2008,"context_line":"                                    headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2009,"context_line":"        if timestamp_str and Timestamp(timestamp_str) \u003e self.timestamp:"},{"line_number":2010,"context_line":"            self.timestamp_str \u003d timestamp_str"},{"line_number":2011,"context_line":"        if not self.gets:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d5da8b77","line":2008,"in_reply_to":"bf51134e_f99cfa6a","updated":"2020-06-30 14:43:40.000000000","message":".meta","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2007,"context_line":"        timestamp_str \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2008,"context_line":"                                    headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2009,"context_line":"        if timestamp_str and Timestamp(timestamp_str) \u003e self.timestamp:"},{"line_number":2010,"context_line":"            self.timestamp_str \u003d timestamp_str"},{"line_number":2011,"context_line":"        if not self.gets:"},{"line_number":2012,"context_line":"            self.status \u003d getter.last_status"},{"line_number":2013,"context_line":"            # stash first set of backend headers, which will be used to"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_5901a669","line":2010,"updated":"2020-06-29 20:27:20.000000000","message":"It feels weird to me that we might update the timestamp on a bucket. I thought that the bucket was supposed to represent a collection of frags that could be used together to generate a response -- which would then imply that all the frags should have the same timestamp. Are we bucketing off of (user-facing) ETag or something?\n\nWhat happens if we get a mix of frags for an 8+4 policy like\n\n* 3 durable frags from t0 with etag1\n* 1 non-durable frag from t2 *also* with etag1\n* 4 durable frags from t1 with etag2, but indicating that they\u0027ve got non-durable frags from t2\n* 4 durable frags from t1 with etag2, no info about t2\n\n? I\u0027d be real nervous about that non-durable frag \"promoting\" the first three frag\u0027s timestamp to make it look like there was durable data at t2.\n\nOr is this supposed to just be for error response buckets? If so, I might prefer to hoist the logic up into add_bad_resp() to make that more clear.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2007,"context_line":"        timestamp_str \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2008,"context_line":"                                    headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2009,"context_line":"        if timestamp_str and Timestamp(timestamp_str) \u003e self.timestamp:"},{"line_number":2010,"context_line":"            self.timestamp_str \u003d timestamp_str"},{"line_number":2011,"context_line":"        if not self.gets:"},{"line_number":2012,"context_line":"            self.status \u003d getter.last_status"},{"line_number":2013,"context_line":"            # stash first set of backend headers, which will be used to"}],"source_content_type":"text/x-python","patch_set":12,"id":"9f560f44_705c127e","line":2010,"in_reply_to":"bf51134e_35d5c744","updated":"2020-08-19 05:39:21.000000000","message":"Note to self: I still want to test this out...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2007,"context_line":"        timestamp_str \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2008,"context_line":"                                    headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2009,"context_line":"        if timestamp_str and Timestamp(timestamp_str) \u003e self.timestamp:"},{"line_number":2010,"context_line":"            self.timestamp_str \u003d timestamp_str"},{"line_number":2011,"context_line":"        if not self.gets:"},{"line_number":2012,"context_line":"            self.status \u003d getter.last_status"},{"line_number":2013,"context_line":"            # stash first set of backend headers, which will be used to"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_35d5c744","line":2010,"in_reply_to":"bf51134e_5901a669","updated":"2020-06-30 14:43:40.000000000","message":"it\u0027s worth thinking about, buckets \"have\" a timestamp whether they explicitly keep track of it or not.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2059,"context_line":"            self.alt_nodes[frag_index].append(node)"},{"line_number":2060,"context_line":""},{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall(self):"},{"line_number":2063,"context_line":"        resp_count \u003d len(self.get_responses())"},{"line_number":2064,"context_line":"        if self.durable or self.status \u003d\u003d 416:"},{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_59eb6676","line":2062,"updated":"2020-06-29 20:27:20.000000000","message":"I feel like this could use a docstring now. It used to be short and straight-forward enough that it maybe wasn\u0027t strictly necessary, but it\u0027s clearly no longer just\n\n Number of additional responses required to be able to reconstruct.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2059,"context_line":"            self.alt_nodes[frag_index].append(node)"},{"line_number":2060,"context_line":""},{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall(self):"},{"line_number":2063,"context_line":"        resp_count \u003d len(self.get_responses())"},{"line_number":2064,"context_line":"        if self.durable or self.status \u003d\u003d 416:"},{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_95cad39e","line":2062,"in_reply_to":"bf51134e_59eb6676","updated":"2020-06-30 14:43:40.000000000","message":"#willfix","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall(self):"},{"line_number":2063,"context_line":"        resp_count \u003d len(self.get_responses())"},{"line_number":2064,"context_line":"        if self.durable or self.status \u003d\u003d 416:"},{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"},{"line_number":2066,"context_line":"        alt_count \u003d min(self.policy.object_ring.replica_count - resp_count,"},{"line_number":2067,"context_line":"                        self.policy.ec_nparity)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f946babf","line":2064,"range":{"start_line":2064,"start_character":42,"end_line":2064,"end_character":45},"updated":"2020-06-29 20:27:20.000000000","message":"Why the status-code exception? Should there be a similar exception for 304 Not Modified or 412 Precondition Failed?\n\nShould there be a similar change in shortfall_with_alts()?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2061,"context_line":"    @property"},{"line_number":2062,"context_line":"    def shortfall(self):"},{"line_number":2063,"context_line":"        resp_count \u003d len(self.get_responses())"},{"line_number":2064,"context_line":"        if self.durable or self.status \u003d\u003d 416:"},{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"},{"line_number":2066,"context_line":"        alt_count \u003d min(self.policy.object_ring.replica_count - resp_count,"},{"line_number":2067,"context_line":"                        self.policy.ec_nparity)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d548cb2f","line":2064,"range":{"start_line":2064,"start_character":42,"end_line":2064,"end_character":45},"in_reply_to":"bf51134e_f946babf","updated":"2020-06-30 14:43:40.000000000","message":"some test fails without it :D\n\nThere\u0027s just a historical precedence of range-not-satisfiable being an early return w/o digging into handoffs.  It\u0027s quite possible there\u0027s a set of responses that should all be treated as athorative.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2064,"context_line":"        if self.durable or self.status \u003d\u003d 416:"},{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"},{"line_number":2066,"context_line":"        alt_count \u003d min(self.policy.object_ring.replica_count - resp_count,"},{"line_number":2067,"context_line":"                        self.policy.ec_nparity)"},{"line_number":2068,"context_line":"        return max([1, self.policy.ec_ndata - resp_count, alt_count])"},{"line_number":2069,"context_line":""},{"line_number":2070,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_99d7bebf","line":2067,"range":{"start_line":2067,"start_character":36,"end_line":2067,"end_character":46},"updated":"2020-06-29 20:27:20.000000000","message":"Right, so we cap it at nparity for the sake of ec frag duplication...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"151f9e148e9f4f818afceeb41d5b146a07465384","unresolved":false,"context_lines":[{"line_number":2065,"context_line":"            return max(self.policy.ec_ndata - resp_count, 0)"},{"line_number":2066,"context_line":"        alt_count \u003d min(self.policy.object_ring.replica_count - resp_count,"},{"line_number":2067,"context_line":"                        self.policy.ec_nparity)"},{"line_number":2068,"context_line":"        return max([1, self.policy.ec_ndata - resp_count, alt_count])"},{"line_number":2069,"context_line":""},{"line_number":2070,"context_line":"    @property"},{"line_number":2071,"context_line":"    def shortfall_with_alts(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d76161d5","line":2068,"updated":"2020-06-22 22:45:11.000000000","message":"this effects the goals outlined in https://etherpad.opendev.org/p/the-non-durable-problem and is supported by the test at https://review.opendev.org/#/c/711342/12/test/unit/proxy/controllers/test_obj.py@4041","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2073,"context_line":"        # for frags on the alt nodes."},{"line_number":2074,"context_line":"        alts \u003d set(self.alt_nodes.keys()).difference(set(self.gets.keys()))"},{"line_number":2075,"context_line":"        result \u003d self.policy.ec_ndata - (len(self.get_responses()) + len(alts))"},{"line_number":2076,"context_line":"        return max(result, 0)"},{"line_number":2077,"context_line":""},{"line_number":2078,"context_line":"    def __str__(self):"},{"line_number":2079,"context_line":"        # return a string summarising bucket state, useful for debugging."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b978c2c3","line":2076,"range":{"start_line":2076,"start_character":27,"end_line":2076,"end_character":28},"updated":"2020-06-29 20:27:20.000000000","message":"Is it expected/desirable that this could go to zero when non-durable, while shortfall() can\u0027t?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2073,"context_line":"        # for frags on the alt nodes."},{"line_number":2074,"context_line":"        alts \u003d set(self.alt_nodes.keys()).difference(set(self.gets.keys()))"},{"line_number":2075,"context_line":"        result \u003d self.policy.ec_ndata - (len(self.get_responses()) + len(alts))"},{"line_number":2076,"context_line":"        return max(result, 0)"},{"line_number":2077,"context_line":""},{"line_number":2078,"context_line":"    def __str__(self):"},{"line_number":2079,"context_line":"        # return a string summarising bucket state, useful for debugging."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_356307b4","line":2076,"range":{"start_line":2076,"start_character":27,"end_line":2076,"end_character":28},"in_reply_to":"bf51134e_b978c2c3","updated":"2020-06-30 14:43:40.000000000","message":"the alt handling is probably under tested","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2085,"context_line":"    def timestamp(self):"},{"line_number":2086,"context_line":"        if not self.timestamp_str:"},{"line_number":2087,"context_line":"            return None"},{"line_number":2088,"context_line":"        return Timestamp(self.timestamp_str)"},{"line_number":2089,"context_line":""},{"line_number":2090,"context_line":""},{"line_number":2091,"context_line":"class ECGetResponseCollection(object):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_191f2e6c","line":2088,"updated":"2020-06-29 20:27:20.000000000","message":"I wonder if we should just have the timestamp attribute rather than re-parse it every time we want to make a comparison. (And add a timestamp_str getter if we really need one, but I suspect we don\u0027t.)","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2085,"context_line":"    def timestamp(self):"},{"line_number":2086,"context_line":"        if not self.timestamp_str:"},{"line_number":2087,"context_line":"            return None"},{"line_number":2088,"context_line":"        return Timestamp(self.timestamp_str)"},{"line_number":2089,"context_line":""},{"line_number":2090,"context_line":""},{"line_number":2091,"context_line":"class ECGetResponseCollection(object):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_848623db","line":2088,"in_reply_to":"bf51134e_156c039b","updated":"2020-06-30 18:58:51.000000000","message":"I\u0027ve not yet, no.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2085,"context_line":"    def timestamp(self):"},{"line_number":2086,"context_line":"        if not self.timestamp_str:"},{"line_number":2087,"context_line":"            return None"},{"line_number":2088,"context_line":"        return Timestamp(self.timestamp_str)"},{"line_number":2089,"context_line":""},{"line_number":2090,"context_line":""},{"line_number":2091,"context_line":"class ECGetResponseCollection(object):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_156c039b","line":2088,"in_reply_to":"bf51134e_191f2e6c","updated":"2020-06-30 14:43:40.000000000","message":"probably not a significant issue; I\u0027d have to try it see if it\u0027s more clear - did you already put a diff that passes tests together?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2106,"context_line":"        \"\"\""},{"line_number":2107,"context_line":"        self.policy \u003d policy"},{"line_number":2108,"context_line":"        self.buckets \u003d {}"},{"line_number":2109,"context_line":"        self.bad_buckets \u003d {None: ECGetResponseBucket(self.policy, None)}"},{"line_number":2110,"context_line":"        self.node_iter_count \u003d 0"},{"line_number":2111,"context_line":""},{"line_number":2112,"context_line":"    def _get_bucket(self, timestamp_str):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_34015706","line":2109,"range":{"start_line":2109,"start_character":28,"end_line":2109,"end_character":32},"updated":"2020-06-29 20:27:20.000000000","message":"So I guess this is for things like Timeouts and socket errors?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2106,"context_line":"        \"\"\""},{"line_number":2107,"context_line":"        self.policy \u003d policy"},{"line_number":2108,"context_line":"        self.buckets \u003d {}"},{"line_number":2109,"context_line":"        self.bad_buckets \u003d {None: ECGetResponseBucket(self.policy, None)}"},{"line_number":2110,"context_line":"        self.node_iter_count \u003d 0"},{"line_number":2111,"context_line":""},{"line_number":2112,"context_line":"    def _get_bucket(self, timestamp_str):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_75129f0b","line":2109,"range":{"start_line":2109,"start_character":28,"end_line":2109,"end_character":32},"in_reply_to":"bf51134e_34015706","updated":"2020-06-30 14:43:40.000000000","message":"500s and 404s from primaries too - all the bad responses pretty sure","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2106,"context_line":"        \"\"\""},{"line_number":2107,"context_line":"        self.policy \u003d policy"},{"line_number":2108,"context_line":"        self.buckets \u003d {}"},{"line_number":2109,"context_line":"        self.bad_buckets \u003d {None: ECGetResponseBucket(self.policy, None)}"},{"line_number":2110,"context_line":"        self.node_iter_count \u003d 0"},{"line_number":2111,"context_line":""},{"line_number":2112,"context_line":"    def _get_bucket(self, timestamp_str):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_c4c65b4a","line":2109,"range":{"start_line":2109,"start_character":28,"end_line":2109,"end_character":32},"in_reply_to":"bf51134e_75129f0b","updated":"2020-06-30 18:58:51.000000000","message":"No, it\u0027s keyed off status, right? I was just wondering about the one bucket we pre-populate here.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2196,"context_line":"        the closest to having sufficient getters, unless it is bettered by a"},{"line_number":2197,"context_line":"        bucket with potential alternate nodes."},{"line_number":2198,"context_line":""},{"line_number":2199,"context_line":"        If there are no good buckets we return the \"worst\" bucket."},{"line_number":2200,"context_line":""},{"line_number":2201,"context_line":"        :return: An instance of :class:`~ECGetResponseBucket` or None if there"},{"line_number":2202,"context_line":"                 are no buckets in the collection."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_54584b2f","line":2199,"updated":"2020-06-29 20:27:20.000000000","message":"Wait, what?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2203,"context_line":"        \"\"\""},{"line_number":2204,"context_line":"        sorted_buckets \u003d self._sort_buckets()"},{"line_number":2205,"context_line":"        for bucket in sorted_buckets:"},{"line_number":2206,"context_line":"            # tombstones will set bad_bucket.timestamp"},{"line_number":2207,"context_line":"            not_found_bucket \u003d self.bad_buckets.get(404)"},{"line_number":2208,"context_line":"            if not_found_bucket and not_found_bucket.timestamp and \\"},{"line_number":2209,"context_line":"                    bucket.timestamp \u003c not_found_bucket.timestamp:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_746e2fc9","line":2206,"updated":"2020-06-29 20:27:20.000000000","message":"This kinda feels like 404-with-timestamp ought to count as \"good\" -- the shortfall calculation gets a little weird, though.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2215,"context_line":"    @property"},{"line_number":2216,"context_line":"    def worst_bucket(self):"},{"line_number":2217,"context_line":"        \"\"\""},{"line_number":2218,"context_line":"        Return the bad_bucket with the smallest shortfall"},{"line_number":2219,"context_line":"        \"\"\""},{"line_number":2220,"context_line":"        # we want \"enough\" 416s to prevent \"extra\" requests - but we keep"},{"line_number":2221,"context_line":"        # digging on 404s"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_3478378e","line":2218,"updated":"2020-06-29 20:27:20.000000000","message":"Ah -- best_bucket\u0027s docstring makes a little more sense now. That sounds more like \"least-bad\", not \"worst\" though :-/","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2215,"context_line":"    @property"},{"line_number":2216,"context_line":"    def worst_bucket(self):"},{"line_number":2217,"context_line":"        \"\"\""},{"line_number":2218,"context_line":"        Return the bad_bucket with the smallest shortfall"},{"line_number":2219,"context_line":"        \"\"\""},{"line_number":2220,"context_line":"        # we want \"enough\" 416s to prevent \"extra\" requests - but we keep"},{"line_number":2221,"context_line":"        # digging on 404s"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f5fd0fb2","line":2218,"in_reply_to":"bf51134e_3478378e","updated":"2020-06-30 14:43:40.000000000","message":"yeah I like that!","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"        # digging on 404s"},{"line_number":2222,"context_line":"        short, status \u003d min((bucket.shortfall, status)"},{"line_number":2223,"context_line":"                            for status, bucket in self.bad_buckets.items()"},{"line_number":2224,"context_line":"                            if status !\u003d 404)"},{"line_number":2225,"context_line":"        return self.bad_buckets[status]"},{"line_number":2226,"context_line":""},{"line_number":2227,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_d4df3b63","line":2224,"range":{"start_line":2224,"start_character":28,"end_line":2224,"end_character":44},"updated":"2020-06-29 20:27:20.000000000","message":"Another argument in favor of putting 404s in self.buckets instead of self.bad_buckets.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"        # digging on 404s"},{"line_number":2222,"context_line":"        short, status \u003d min((bucket.shortfall, status)"},{"line_number":2223,"context_line":"                            for status, bucket in self.bad_buckets.items()"},{"line_number":2224,"context_line":"                            if status !\u003d 404)"},{"line_number":2225,"context_line":"        return self.bad_buckets[status]"},{"line_number":2226,"context_line":""},{"line_number":2227,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_e47b9f26","line":2224,"range":{"start_line":2224,"start_character":28,"end_line":2224,"end_character":44},"in_reply_to":"bf51134e_b51c3714","updated":"2020-06-30 18:58:51.000000000","message":"If we think of a bucket as \"waiting for enough responses to send a client response\" instead of \"waiting for enough responses to rebuild\" ... maybe both 404 (with timestamp) and 416 should be considered \"good\".","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"        # digging on 404s"},{"line_number":2222,"context_line":"        short, status \u003d min((bucket.shortfall, status)"},{"line_number":2223,"context_line":"                            for status, bucket in self.bad_buckets.items()"},{"line_number":2224,"context_line":"                            if status !\u003d 404)"},{"line_number":2225,"context_line":"        return self.bad_buckets[status]"},{"line_number":2226,"context_line":""},{"line_number":2227,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b51c3714","line":2224,"range":{"start_line":2224,"start_character":28,"end_line":2224,"end_character":44},"in_reply_to":"bf51134e_d4df3b63","updated":"2020-06-30 14:43:40.000000000","message":"i don\u0027t follow, it\u0027s not obvious to me how to reconsider best_bucket as something that might not rebuild...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cd572cdac058ab1dd82acc2745af0b4d9cd0c3f8","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"        # digging on 404s"},{"line_number":2222,"context_line":"        short, status \u003d min((bucket.shortfall, status)"},{"line_number":2223,"context_line":"                            for status, bucket in self.bad_buckets.items()"},{"line_number":2224,"context_line":"                            if status !\u003d 404)"},{"line_number":2225,"context_line":"        return self.bad_buckets[status]"},{"line_number":2226,"context_line":""},{"line_number":2227,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_0a685db8","line":2224,"range":{"start_line":2224,"start_character":28,"end_line":2224,"end_character":44},"in_reply_to":"bf51134e_e47b9f26","updated":"2020-06-30 19:38:41.000000000","message":"I\u0027ve already unified some handling of buckets vs bad_buckets through best/worst_bucket; i don\u0027t even think there\u0027s an attribute that explicitly calls a bucket good/bad\n\nYou\u0027re probably on track to a significant insight here that\u0027s going to make a lot of these ugly exceptional conditions and magic numbers go away... but it might be over my head.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2300,"context_line":"            return nodes.pop(0).copy()"},{"line_number":2301,"context_line":""},{"line_number":2302,"context_line":""},{"line_number":2303,"context_line":"def is_good_source(status):"},{"line_number":2304,"context_line":"    \"\"\""},{"line_number":2305,"context_line":"    Indicates whether or not the request made to the backend found"},{"line_number":2306,"context_line":"    what it was looking for."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_e4a0bf94","line":2303,"updated":"2020-06-30 18:58:51.000000000","message":"It\u0027s a little weird that we can have is_good_source() return True but then we put it in a \"bad\" bucket :-/","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cd572cdac058ab1dd82acc2745af0b4d9cd0c3f8","unresolved":false,"context_lines":[{"line_number":2300,"context_line":"            return nodes.pop(0).copy()"},{"line_number":2301,"context_line":""},{"line_number":2302,"context_line":""},{"line_number":2303,"context_line":"def is_good_source(status):"},{"line_number":2304,"context_line":"    \"\"\""},{"line_number":2305,"context_line":"    Indicates whether or not the request made to the backend found"},{"line_number":2306,"context_line":"    what it was looking for."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_8ae74d47","line":2303,"in_reply_to":"bf51134e_e4a0bf94","updated":"2020-06-30 19:38:41.000000000","message":"yeah this sucks :\u0027(\n\nmaybe \"good/bad/worst\" is just the wrong concepts\n\nbuckets_by_timestamp\nbuckets_by_status","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2441,"context_line":"    def response_parts_iter(self, req):"},{"line_number":2442,"context_line":"        try:"},{"line_number":2443,"context_line":"            source, node \u003d self._get_source_and_node()"},{"line_number":2444,"context_line":"        except StopIteration:"},{"line_number":2445,"context_line":"            return"},{"line_number":2446,"context_line":"        it \u003d None"},{"line_number":2447,"context_line":"        if source:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_143373ac","line":2444,"updated":"2020-06-29 20:27:20.000000000","message":"Funny -- why wouldn\u0027t we have hit this before?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2441,"context_line":"    def response_parts_iter(self, req):"},{"line_number":2442,"context_line":"        try:"},{"line_number":2443,"context_line":"            source, node \u003d self._get_source_and_node()"},{"line_number":2444,"context_line":"        except StopIteration:"},{"line_number":2445,"context_line":"            return"},{"line_number":2446,"context_line":"        it \u003d None"},{"line_number":2447,"context_line":"        if source:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_75e0dffb","line":2444,"in_reply_to":"bf51134e_143373ac","updated":"2020-06-30 14:43:40.000000000","message":"something to do with using for node in node iter insider _get_source_and_node","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2455,"context_line":""},{"line_number":2456,"context_line":"        try:"},{"line_number":2457,"context_line":"            client_chunk_size \u003d self.client_chunk_size"},{"line_number":2458,"context_line":"            node_timeout \u003d self.app.node_timeout"},{"line_number":2459,"context_line":"            node_timeout \u003d self.app.recoverable_node_timeout"},{"line_number":2460,"context_line":""},{"line_number":2461,"context_line":"            # This is safe; it sets up a generator but does not call next()"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_54296b34","line":2458,"range":{"start_line":2458,"start_character":12,"end_line":2458,"end_character":48},"updated":"2020-06-29 20:27:20.000000000","message":"We can drop this now.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2455,"context_line":""},{"line_number":2456,"context_line":"        try:"},{"line_number":2457,"context_line":"            client_chunk_size \u003d self.client_chunk_size"},{"line_number":2458,"context_line":"            node_timeout \u003d self.app.node_timeout"},{"line_number":2459,"context_line":"            node_timeout \u003d self.app.recoverable_node_timeout"},{"line_number":2460,"context_line":""},{"line_number":2461,"context_line":"            # This is safe; it sets up a generator but does not call next()"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b5e1d7ff","line":2458,"range":{"start_line":2458,"start_character":12,"end_line":2458,"end_character":48},"in_reply_to":"bf51134e_54296b34","updated":"2020-06-30 14:43:40.000000000","message":"#willfix","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2483,"context_line":"                                parts_iter[0])"},{"line_number":2484,"context_line":"                        return (start_byte, end_byte, length, headers, part)"},{"line_number":2485,"context_line":"                    except ChunkReadTimeout:"},{"line_number":2486,"context_line":"                        new_source, new_node \u003d self._dig_for_source_and_node()"},{"line_number":2487,"context_line":"                        if new_source:"},{"line_number":2488,"context_line":"                            self.app.error_occurred("},{"line_number":2489,"context_line":"                                node[0], _(\u0027Trying to read object during \u0027"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_74b46ff2","line":2486,"updated":"2020-06-29 20:27:20.000000000","message":"It feels a little weird that we catch the StopIteration in _get_source_and_node() just to raise a fresh one here -- but I guess it\u0027s so we know we\u0027ve got a \"good\" response.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2483,"context_line":"                                parts_iter[0])"},{"line_number":2484,"context_line":"                        return (start_byte, end_byte, length, headers, part)"},{"line_number":2485,"context_line":"                    except ChunkReadTimeout:"},{"line_number":2486,"context_line":"                        new_source, new_node \u003d self._dig_for_source_and_node()"},{"line_number":2487,"context_line":"                        if new_source:"},{"line_number":2488,"context_line":"                            self.app.error_occurred("},{"line_number":2489,"context_line":"                                node[0], _(\u0027Trying to read object during \u0027"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_15efe3e9","line":2486,"in_reply_to":"bf51134e_74b46ff2","updated":"2020-06-30 14:43:40.000000000","message":"i tried a couple of times to move the StopIteration exceptions around and couldn\u0027t find a way that worked and made it more clear.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2766,"context_line":"                hdrs \u003d HeaderKeyDict(possible_source.getheaders())"},{"line_number":2767,"context_line":"                ts \u003d Timestamp(hdrs.get(\u0027X-Backend-Timestamp\u0027, 0))"},{"line_number":2768,"context_line":"                if ts \u003e self.latest_404_timestamp:"},{"line_number":2769,"context_line":"                    self.latest_404_timestamp \u003d ts"},{"line_number":2770,"context_line":"            if possible_source.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":2771,"context_line":"                self.app.error_limit(node, _(\u0027ERROR Insufficient Storage\u0027))"},{"line_number":2772,"context_line":"            elif is_server_error(possible_source.status):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_14e253d5","line":2769,"updated":"2020-06-29 20:27:20.000000000","message":"This feels similar to what\u0027s going on with the response bucketing -- any way we could combine things a bit?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2766,"context_line":"                hdrs \u003d HeaderKeyDict(possible_source.getheaders())"},{"line_number":2767,"context_line":"                ts \u003d Timestamp(hdrs.get(\u0027X-Backend-Timestamp\u0027, 0))"},{"line_number":2768,"context_line":"                if ts \u003e self.latest_404_timestamp:"},{"line_number":2769,"context_line":"                    self.latest_404_timestamp \u003d ts"},{"line_number":2770,"context_line":"            if possible_source.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":2771,"context_line":"                self.app.error_limit(node, _(\u0027ERROR Insufficient Storage\u0027))"},{"line_number":2772,"context_line":"            elif is_server_error(possible_source.status):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_559f9b64","line":2769,"in_reply_to":"bf51134e_14e253d5","updated":"2020-06-30 14:43:40.000000000","message":"this is probably dead weight - FragGetters no longer gather multiple responses... except in the resuming case","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2786,"context_line":""},{"line_number":2787,"context_line":"        node_timeout \u003d self.app.recoverable_node_timeout"},{"line_number":2788,"context_line":""},{"line_number":2789,"context_line":"        # XXX this is the big change, we\u0027ve removed the concurrency and let"},{"line_number":2790,"context_line":"        # 404/503 bubble out to the caller immediately; all the control is"},{"line_number":2791,"context_line":"        # handled above this"},{"line_number":2792,"context_line":"        # ... depending on how it shakes out we\u0027ll probably drop the \"append"},{"line_number":2793,"context_line":"        # statuses to list of all responses\" interface"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_342dd7fd","line":2790,"range":{"start_line":2789,"start_character":72,"end_line":2790,"end_character":54},"updated":"2020-06-29 20:27:20.000000000","message":"Oh -- or do we just ax all of this latest_404_timestamp business?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2786,"context_line":""},{"line_number":2787,"context_line":"        node_timeout \u003d self.app.recoverable_node_timeout"},{"line_number":2788,"context_line":""},{"line_number":2789,"context_line":"        # XXX this is the big change, we\u0027ve removed the concurrency and let"},{"line_number":2790,"context_line":"        # 404/503 bubble out to the caller immediately; all the control is"},{"line_number":2791,"context_line":"        # handled above this"},{"line_number":2792,"context_line":"        # ... depending on how it shakes out we\u0027ll probably drop the \"append"},{"line_number":2793,"context_line":"        # statuses to list of all responses\" interface"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_95a93392","line":2790,"range":{"start_line":2789,"start_character":72,"end_line":2790,"end_character":54},"in_reply_to":"bf51134e_342dd7fd","updated":"2020-06-30 14:43:40.000000000","message":"yeah if we want to go the fully decoupled route there\u0027s more code to rip out here.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2810,"context_line":"                (k.lower(), v) for k, v in"},{"line_number":2811,"context_line":"                source.getheaders())"},{"line_number":2812,"context_line":""},{"line_number":2813,"context_line":"            # Save off the source etag so that, if we lose the connection"},{"line_number":2814,"context_line":"            # and have to resume from a different node, we can be sure that"},{"line_number":2815,"context_line":"            # we have the same object (replication) or a fragment archive"},{"line_number":2816,"context_line":"            # from the same object (EC). Otherwise, if the cluster has two"},{"line_number":2817,"context_line":"            # versions of the same object, we might end up switching between"},{"line_number":2818,"context_line":"            # old and new mid-stream and giving garbage to the client."},{"line_number":2819,"context_line":"            self.used_source_etag \u003d normalize_etag(src_headers.get("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f446bfbf","line":2816,"range":{"start_line":2813,"start_character":14,"end_line":2816,"end_character":39},"updated":"2020-06-29 20:27:20.000000000","message":"This *is* still relevant to the controlling layer above us, right? Where\u0027s that getting handled?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2810,"context_line":"                (k.lower(), v) for k, v in"},{"line_number":2811,"context_line":"                source.getheaders())"},{"line_number":2812,"context_line":""},{"line_number":2813,"context_line":"            # Save off the source etag so that, if we lose the connection"},{"line_number":2814,"context_line":"            # and have to resume from a different node, we can be sure that"},{"line_number":2815,"context_line":"            # we have the same object (replication) or a fragment archive"},{"line_number":2816,"context_line":"            # from the same object (EC). Otherwise, if the cluster has two"},{"line_number":2817,"context_line":"            # versions of the same object, we might end up switching between"},{"line_number":2818,"context_line":"            # old and new mid-stream and giving garbage to the client."},{"line_number":2819,"context_line":"            self.used_source_etag \u003d normalize_etag(src_headers.get("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_358dc731","line":2816,"range":{"start_line":2813,"start_character":14,"end_line":2816,"end_character":39},"in_reply_to":"bf51134e_f446bfbf","updated":"2020-06-30 14:43:40.000000000","message":"yeah trying to pull out the code for resumming download with out keeping this behavior will blow up in our face.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2817,"context_line":"            # versions of the same object, we might end up switching between"},{"line_number":2818,"context_line":"            # old and new mid-stream and giving garbage to the client."},{"line_number":2819,"context_line":"            self.used_source_etag \u003d normalize_etag(src_headers.get("},{"line_number":2820,"context_line":"                \u0027x-object-sysmeta-ec-etag\u0027, src_headers.get(\u0027etag\u0027, \u0027\u0027)))"},{"line_number":2821,"context_line":"            self.node \u003d node"},{"line_number":2822,"context_line":"            return source, node"},{"line_number":2823,"context_line":"        return None, None"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f4d87f1e","line":2820,"range":{"start_line":2820,"start_character":16,"end_line":2820,"end_character":42},"updated":"2020-06-29 20:27:20.000000000","message":"This is the only ETag header we care about now, yeah?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2906,"context_line":"            try:"},{"line_number":2907,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2908,"context_line":"            except Empty:"},{"line_number":2909,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2910,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2911,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2912,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_1467f34e","line":2909,"range":{"start_line":2909,"start_character":19,"end_line":2909,"end_character":40},"updated":"2020-06-29 20:27:20.000000000","message":"Reaching down into something called \"unsafe_iter\" gave me pause -- I wonder if we should add something like\n\n @property\n def has_primaries_left(self):\n     return bool(self.unsafe_iter.primaries_left)\n\nto GreenthreadSafeIterator...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2906,"context_line":"            try:"},{"line_number":2907,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2908,"context_line":"            except Empty:"},{"line_number":2909,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2910,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2911,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2912,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f5722f49","line":2909,"range":{"start_line":2909,"start_character":19,"end_line":2909,"end_character":40},"in_reply_to":"bf51134e_1467f34e","updated":"2020-06-30 14:43:40.000000000","message":"yuk, maybe - I think NodeIter should probably just be made safe so we can pass it around and access the attributes directly","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2908,"context_line":"            except Empty:"},{"line_number":2909,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2910,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2911,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2912,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2913,"context_line":"                               req, safe_iter, partition,"},{"line_number":2914,"context_line":"                               policy, buckets.get_extra_headers)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_74cf4f28","line":2911,"updated":"2020-06-29 20:27:20.000000000","message":"I think we could find out earlier if we change the\n\n else:\n\nbelow to be\n\n if not safe_iter.unsafe_iter.primaries_left:\n\nbut I suppose it makes things a bit harder to reason about.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2908,"context_line":"            except Empty:"},{"line_number":2909,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2910,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2911,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2912,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2913,"context_line":"                               req, safe_iter, partition,"},{"line_number":2914,"context_line":"                               policy, buckets.get_extra_headers)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b59d5759","line":2911,"in_reply_to":"bf51134e_74cf4f28","updated":"2020-06-30 14:43:40.000000000","message":"there\u0027s a lot of tests that are picky about the eventlet stuff so i\u0027m not sure how much flexibility we have - I think this is fairly clear.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2908,"context_line":"            except Empty:"},{"line_number":2909,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2910,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2911,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2912,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2913,"context_line":"                               req, safe_iter, partition,"},{"line_number":2914,"context_line":"                               policy, buckets.get_extra_headers)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_64baefe1","line":2911,"in_reply_to":"bf51134e_b59d5759","updated":"2020-06-30 18:58:51.000000000","message":"K -- something about the phrasing made me think this was in some way regrettable, but I\u0027m fine with leaving it.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2973,"context_line":"                except ValueError as err:"},{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_0f040014","line":2976,"updated":"2020-06-29 20:27:20.000000000","message":"OK, so the fact that best_bucket may have a \"bad\" response makes for a pretty decent change in mentality here...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2973,"context_line":"                except ValueError as err:"},{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_95b013e0","line":2976,"in_reply_to":"bf51134e_0f040014","updated":"2020-06-30 14:43:40.000000000","message":"what\u0027d you call it?  \"least_bad\" - it\u0027s like \"best of the worst\" or something...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"},{"line_number":2980,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_ef1a6c6c","line":2977,"range":{"start_line":2977,"start_character":31,"end_line":2977,"end_character":38},"updated":"2020-06-29 20:27:20.000000000","message":"Note that previously, the (single) bucket full of bad responses was always marked durable, but now none of the bad buckets ever are.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"},{"line_number":2980,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_c4e13bff","line":2977,"range":{"start_line":2977,"start_character":31,"end_line":2977,"end_character":38},"in_reply_to":"bf51134e_b5cb9748","updated":"2020-06-30 18:58:51.000000000","message":"IDK -- I could see an argument that if we get back ndata 404s (especially if they all had timestamps!) there\u0027s no need to dig into handoffs...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cd572cdac058ab1dd82acc2745af0b4d9cd0c3f8","unresolved":false,"context_lines":[{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"},{"line_number":2980,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_0a117d54","line":2977,"range":{"start_line":2977,"start_character":31,"end_line":2977,"end_character":38},"in_reply_to":"bf51134e_c4e13bff","updated":"2020-06-30 19:38:41.000000000","message":"oh yeah, wow... that could be a big deal... i\u0027d love to play with that more","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":2974,"context_line":"                    self.app.logger.error("},{"line_number":2975,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2976,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2977,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2978,"context_line":"                    # good enough!"},{"line_number":2979,"context_line":"                    break"},{"line_number":2980,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b5cb9748","line":2977,"range":{"start_line":2977,"start_character":31,"end_line":2977,"end_character":38},"in_reply_to":"bf51134e_ef1a6c6c","updated":"2020-06-30 14:43:40.000000000","message":"it\u0027s not obvious to me that bad buckets should or shouldn\u0027t be durable - I did change it to get rid of some conditions.\n\nAs far as this break I think there\u0027s never \"enough\" bad responses - there\u0027s just \"ran out of reasonable requests to try\"","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":3023,"context_line":"                resp \u003d err_resp"},{"line_number":3024,"context_line":"        else:"},{"line_number":3025,"context_line":"            # TODO: we can get here if all buckets are successful but none"},{"line_number":3026,"context_line":"            # have ec_ndata getters, so bad_bucket may have no gets and we will"},{"line_number":3027,"context_line":"            # return a 503 when a 404 may be more appropriate. We can also get"},{"line_number":3028,"context_line":"            # here with less than ec_ndata 416\u0027s and may then return a 416"},{"line_number":3029,"context_line":"            # which is also questionable because a non-range get for same"},{"line_number":3030,"context_line":"            # object would return 404 or 503."}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_4f71d8a7","line":3027,"range":{"start_line":3026,"start_character":72,"end_line":3027,"end_character":61},"updated":"2020-06-29 20:27:20.000000000","message":"Off-topic: I *think* this comment is out of date now that we have https://github.com/openstack/swift/commit/3189410f. Might be wrong, though.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":2302,"context_line":""},{"line_number":2303,"context_line":"def is_good_source(status):"},{"line_number":2304,"context_line":"    \"\"\""},{"line_number":2305,"context_line":"    Indicates whether or not the request made to the backend found"},{"line_number":2306,"context_line":"    what it was looking for."},{"line_number":2307,"context_line":""},{"line_number":2308,"context_line":"    :param status: the response from the backend"},{"line_number":2309,"context_line":"    :returns: True if found, False if not"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_0ea168aa","line":2306,"range":{"start_line":2305,"start_character":61,"end_line":2306,"end_character":27},"updated":"2020-07-29 01:42:38.000000000","message":"This seems *so* squishy -- given this phrasing, it almost seems like we want something like\n\n return headers.get(\u0027x-backend-timestamp\u0027) and not is_server_error(status)\n\n... at least, when we\u0027re still trying to establish quorum. And then we use timestamps and standard quorum detection to decide how to reply. Once we start digging, we *need*\n\n return is_success(status) and headers.get(\n     \u0027x-object-sysmeta-ec-etag\u0027) \u003d\u003d used_source_etag\n\nAny other response at that point is at best worthless, and at worst, corrupting.","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":2440,"context_line":""},{"line_number":2441,"context_line":"    def response_parts_iter(self, req):"},{"line_number":2442,"context_line":"        try:"},{"line_number":2443,"context_line":"            source, node \u003d self._get_source_and_node()"},{"line_number":2444,"context_line":"        except StopIteration:"},{"line_number":2445,"context_line":"            return"},{"line_number":2446,"context_line":"        it \u003d None"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_ba2b5792","line":2443,"updated":"2020-07-29 01:42:38.000000000","message":"I\u0027m still trying to get a feel for when we should get and when we should dig... I guess, digging is just for when we\u0027ve already told a client \"yeah, we\u0027ve got that -- here\u0027s your bytes!\" but then one of the streams dies?","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":2955,"context_line":"            feeder_q \u003d None"},{"line_number":2956,"context_line":"            if self.app.concurrent_gets:"},{"line_number":2957,"context_line":"                feeder_q \u003d Queue()"},{"line_number":2958,"context_line":"                pool.spawn(self.feed_remaining_primaries, safe_iter, pile, req,"},{"line_number":2959,"context_line":"                           partition, policy, buckets, feeder_q)"},{"line_number":2960,"context_line":""},{"line_number":2961,"context_line":"            extra_requests \u003d 0"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_ee67d424","line":2958,"range":{"start_line":2958,"start_character":21,"end_line":2958,"end_character":26},"updated":"2020-07-29 01:42:38.000000000","message":"Hmm... so this blocks and won\u0027t start the feeder until the first response comes back...\n\nWhich is fair, I suppose -- otherwise the feeder\u0027s just gonna block when *it* goes to spawn. I\u0027m just trying to think through how this behaves when *everybody* is slow.","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"37546ea3a40e0d062c57be39c098046e457c4e5e","unresolved":false,"context_lines":[{"line_number":3032,"context_line":"            reasons \u003d []"},{"line_number":3033,"context_line":"            bodies \u003d []"},{"line_number":3034,"context_line":"            headers \u003d []"},{"line_number":3035,"context_line":"            for status, bad_bucket in buckets.bad_buckets.items():"},{"line_number":3036,"context_line":"                for getter, _parts_iter in bad_bucket.get_responses():"},{"line_number":3037,"context_line":"                    if best_bucket and best_bucket.durable:"},{"line_number":3038,"context_line":"                        bad_resp_headers \u003d HeaderKeyDict(getter.last_headers)"}],"source_content_type":"text/x-python","patch_set":14,"id":"9f560f44_4e0ec08d","line":3035,"range":{"start_line":3035,"start_character":16,"end_line":3035,"end_character":22},"updated":"2020-07-29 01:42:38.000000000","message":"nit: We aren\u0027t actually using this, are we?\n\n for bad_bucket in buckets.bad_buckets.values():","commit_id":"9a1fa68daa6968fbec0578847f8daf429abea6d4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"460598e416be769e5f4e6555007e689cf30e9dd1","unresolved":false,"context_lines":[{"line_number":2022,"context_line":"            # recent metadata. We could alternatively choose to the *newest*"},{"line_number":2023,"context_line":"            # metadata headers for self.headers by selecting the source with"},{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_90730fe7","line":2025,"updated":"2020-08-11 06:32:46.000000000","message":"We pull out `last_headers` as headers on line 2007. So shouldn\u0027t we just use headers here?","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"460598e416be769e5f4e6555007e689cf30e9dd1","unresolved":false,"context_lines":[{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"},{"line_number":2030,"context_line":"            # expected. If somehow it happens then ignore those fragments"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_905caf4b","line":2027,"updated":"2020-08-11 06:32:46.000000000","message":"And here.. though I guess `getter.last_headers` here actually makes the code more readable.. so I\u0027m conflicted here :)","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"},{"line_number":2030,"context_line":"            # expected. If somehow it happens then ignore those fragments"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_3111c905","line":2027,"in_reply_to":"9f560f44_66f680fc","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ef993d159809ba759406984efa3eef3ef7ab6d69","unresolved":false,"context_lines":[{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"},{"line_number":2030,"context_line":"            # expected. If somehow it happens then ignore those fragments"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_66f680fc","line":2027,"in_reply_to":"9f560f44_905caf4b","updated":"2020-08-11 15:40:20.000000000","message":"oh yeah, that\u0027s super weird/ugly #willfix","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b4ea8146abf53e4fa76d2d85eed7386a61c7d485","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"                # \"good bucket\" is trumped by newer tombstone"},{"line_number":2222,"context_line":"                continue"},{"line_number":2223,"context_line":"            return bucket"},{"line_number":2224,"context_line":"        return self.least_bad_bucket"},{"line_number":2225,"context_line":""},{"line_number":2226,"context_line":"    @property"},{"line_number":2227,"context_line":"    def least_bad_bucket(self):"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_8e7c5821","line":2224,"updated":"2020-08-14 21:45:48.000000000","message":"OK, so now best_bucket *always* returns not-None, and it might be a \"bad\" bucket...","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"460598e416be769e5f4e6555007e689cf30e9dd1","unresolved":false,"context_lines":[{"line_number":2685,"context_line":"        if self.source_headers:"},{"line_number":2686,"context_line":"            return HeaderKeyDict(self.source_headers)"},{"line_number":2687,"context_line":"        else:"},{"line_number":2688,"context_line":"            return {}"},{"line_number":2689,"context_line":""},{"line_number":2690,"context_line":"    def _make_node_request(self, node, node_timeout, logger_thread_locals):"},{"line_number":2691,"context_line":"        self.app.logger.thread_locals \u003d logger_thread_locals"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_90214fb7","line":2688,"updated":"2020-08-11 06:32:46.000000000","message":"should this just be a dict or an empty HeaderKeyDict, incase anyone ever decides to modify something in last headers (and we want keep the headerkeydict case magic etc)","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2685,"context_line":"        if self.source_headers:"},{"line_number":2686,"context_line":"            return HeaderKeyDict(self.source_headers)"},{"line_number":2687,"context_line":"        else:"},{"line_number":2688,"context_line":"            return {}"},{"line_number":2689,"context_line":""},{"line_number":2690,"context_line":"    def _make_node_request(self, node, node_timeout, logger_thread_locals):"},{"line_number":2691,"context_line":"        self.app.logger.thread_locals \u003d logger_thread_locals"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_b16799b7","line":2688,"in_reply_to":"9f560f44_46d09c54","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ef993d159809ba759406984efa3eef3ef7ab6d69","unresolved":false,"context_lines":[{"line_number":2685,"context_line":"        if self.source_headers:"},{"line_number":2686,"context_line":"            return HeaderKeyDict(self.source_headers)"},{"line_number":2687,"context_line":"        else:"},{"line_number":2688,"context_line":"            return {}"},{"line_number":2689,"context_line":""},{"line_number":2690,"context_line":"    def _make_node_request(self, node, node_timeout, logger_thread_locals):"},{"line_number":2691,"context_line":"        self.app.logger.thread_locals \u003d logger_thread_locals"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_46d09c54","line":2688,"in_reply_to":"9f560f44_90214fb7","updated":"2020-08-11 15:40:20.000000000","message":"type consistency seems like a reasonable improvement to me!  #willfix","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"460598e416be769e5f4e6555007e689cf30e9dd1","unresolved":false,"context_lines":[{"line_number":2845,"context_line":"            try:"},{"line_number":2846,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2847,"context_line":"            except Empty:"},{"line_number":2848,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2849,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2850,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2851,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_7012bb63","line":2848,"updated":"2020-08-11 06:32:46.000000000","message":"lol, safe_iter.unsafe_iter !! Not saying it\u0027s wrong just looks.. unsafe?? I\u0027m guessing a safe iter is wrapping something unsafe.","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ef993d159809ba759406984efa3eef3ef7ab6d69","unresolved":false,"context_lines":[{"line_number":2845,"context_line":"            try:"},{"line_number":2846,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2847,"context_line":"            except Empty:"},{"line_number":2848,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2849,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2850,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2851,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_066ce49a","line":2848,"in_reply_to":"9f560f44_7012bb63","updated":"2020-08-11 15:40:20.000000000","message":"yeah i hate this code so much - thanks for pointing it out.  \n\nI think the long term strategy is to make the node iter itself green-safe (I had a spike that did that, but never pushed it up because I felt it deserved more unittests than I was going to write at the time).\n\nOther options I considered:\n  * pass both the safe_iter and the node_iter\n  * add getattr passthrough to wrapped safe iters\n\nBut this is was the \"most obvious\" compromise I settled on.","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8e3f96c92c79201678bb1add6e55f25e0dbb15b9","unresolved":false,"context_lines":[{"line_number":2975,"context_line":"            headers \u003d []"},{"line_number":2976,"context_line":"            for status, bad_bucket in buckets.bad_buckets.items():"},{"line_number":2977,"context_line":"                for getter, _parts_iter in bad_bucket.get_responses():"},{"line_number":2978,"context_line":"                    if best_bucket and best_bucket.durable:"},{"line_number":2979,"context_line":"                        bad_resp_headers \u003d getter.last_headers"},{"line_number":2980,"context_line":"                        t_data_file \u003d bad_resp_headers.get("},{"line_number":2981,"context_line":"                            \u0027X-Backend-Data-Timestamp\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_32c5998f","line":2978,"updated":"2020-08-17 19:50:37.000000000","message":"I think here we\u0027re also assuming best_bucket.durable is_success; which is maybe reasonable?  bad_buckets do seem to always be durable \u003d\u003d False","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b4ea8146abf53e4fa76d2d85eed7386a61c7d485","unresolved":false,"context_lines":[{"line_number":2993,"context_line":"                        bodies.append(getter.body)"},{"line_number":2994,"context_line":"                        headers.append(getter.source_headers)"},{"line_number":2995,"context_line":""},{"line_number":2996,"context_line":"            if not statuses and best_bucket and not best_bucket.durable:"},{"line_number":2997,"context_line":"                # pretend that non-durable bucket was 404s"},{"line_number":2998,"context_line":"                statuses.append(404)"},{"line_number":2999,"context_line":"                reasons.append(\u0027404 Not Found\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_ee92344b","line":2996,"range":{"start_line":2996,"start_character":28,"end_line":2996,"end_character":71},"updated":"2020-08-14 21:45:48.000000000","message":"... so this will always be True (since \"bad\" buckets are never durable).\n\nWon\u0027t this mean we return 404s when 503 would\u0027ve been more appropriate? Probably just need to replace the\n\n and best bucket\n\nwith\n\n and is_success(best_bucket.status)\n\nor something like that...","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aab5eb0136848d0c82165b85fe08d12c1c20a1d","unresolved":false,"context_lines":[{"line_number":2993,"context_line":"                        bodies.append(getter.body)"},{"line_number":2994,"context_line":"                        headers.append(getter.source_headers)"},{"line_number":2995,"context_line":""},{"line_number":2996,"context_line":"            if not statuses and best_bucket and not best_bucket.durable:"},{"line_number":2997,"context_line":"                # pretend that non-durable bucket was 404s"},{"line_number":2998,"context_line":"                statuses.append(404)"},{"line_number":2999,"context_line":"                reasons.append(\u0027404 Not Found\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_71f54eed","line":2996,"range":{"start_line":2996,"start_character":28,"end_line":2996,"end_character":71},"in_reply_to":"9f560f44_b2e88916","updated":"2020-08-17 21:59:59.000000000","message":"Yeah, and treating the non-durable 200s as 404s seems reasonable, since they don\u0027t have any other data there. (If they did, they would\u0027ve returned the durable data *first*, mentioned the non-durable data was available, and waited for the proxy to explicitly request the non-durable data.)\n\nIf they aren\u0027t successful responses, though, this doesn\u0027t seem right:\n\n vagrant@saio:~/swift$ swift-init main status\n account-server running (23133 - /etc/swift/account-server/1.conf.d)\n account-server running (23134 - /etc/swift/account-server/2.conf.d)\n account-server running (23135 - /etc/swift/account-server/3.conf.d)\n account-server running (23136 - /etc/swift/account-server/4.conf.d)\n container-server running (23129 - /etc/swift/container-server/1.conf.d)\n container-server running (23130 - /etc/swift/container-server/2.conf.d)\n container-server running (23131 - /etc/swift/container-server/3.conf.d)\n container-server running (23132 - /etc/swift/container-server/4.conf.d)\n proxy-server running (23123 - /etc/swift/proxy-server/proxy-noauth.conf.d)\n proxy-server running (23124 - /etc/swift/proxy-server/proxy-server.conf.d)\n No object-server running\n vagrant@saio:~/swift$ swift download c tox.ini --no-download\n Object \u0027c/tox.ini\u0027 not found","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8e3f96c92c79201678bb1add6e55f25e0dbb15b9","unresolved":false,"context_lines":[{"line_number":2993,"context_line":"                        bodies.append(getter.body)"},{"line_number":2994,"context_line":"                        headers.append(getter.source_headers)"},{"line_number":2995,"context_line":""},{"line_number":2996,"context_line":"            if not statuses and best_bucket and not best_bucket.durable:"},{"line_number":2997,"context_line":"                # pretend that non-durable bucket was 404s"},{"line_number":2998,"context_line":"                statuses.append(404)"},{"line_number":2999,"context_line":"                reasons.append(\u0027404 Not Found\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9f560f44_b2e88916","line":2996,"range":{"start_line":2996,"start_character":28,"end_line":2996,"end_character":71},"in_reply_to":"9f560f44_ee92344b","updated":"2020-08-17 19:50:37.000000000","message":"tests seem to show we get here where bad_buckets is essentially \"empty\" (all nodes returned frags; we just couldn\u0027t put them together into a response)\n\n... in such a case the question of returning \"503 because []\" or \"404\" mostly seems to be \"was the best of the unfinished buckets\" durable or not","commit_id":"dbf1afcfccd2cf0a83db2520691febf5728a3998"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2674,"context_line":"            possible_source.getheaders())"},{"line_number":2675,"context_line":"        if self.is_good_source(possible_source):"},{"line_number":2676,"context_line":"            # 404 if we know we don\u0027t have a synced copy"},{"line_number":2677,"context_line":"            if not float(possible_source.getheader(\u0027X-PUT-Timestamp\u0027, 1)):"},{"line_number":2678,"context_line":"                self.statuses.append(HTTP_NOT_FOUND)"},{"line_number":2679,"context_line":"                self.reasons.append(\u0027\u0027)"},{"line_number":2680,"context_line":"                self.bodies.append(\u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_eaf033cc","side":"PARENT","line":2677,"range":{"start_line":2677,"start_character":52,"end_line":2677,"end_character":67},"updated":"2020-08-19 05:39:21.000000000","message":"OK, right -- this is a A/C-only header, so the following block was never hit.","commit_id":"5b2c846c69cba6dc707ff32318b715791711f34e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2698,"context_line":"                    src_headers.get(\u0027x-backend-timestamp\u0027) or"},{"line_number":2699,"context_line":"                    src_headers.get(\u0027x-put-timestamp\u0027) or"},{"line_number":2700,"context_line":"                    src_headers.get(\u0027x-timestamp\u0027) or 0)"},{"line_number":2701,"context_line":"                if ps_timestamp \u003e\u003d self.latest_404_timestamp:"},{"line_number":2702,"context_line":"                    self.statuses.append(possible_source.status)"},{"line_number":2703,"context_line":"                    self.reasons.append(possible_source.reason)"},{"line_number":2704,"context_line":"                    self.bodies.append(None)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_6ab6e3fc","side":"PARENT","line":2701,"updated":"2020-08-19 05:39:21.000000000","message":"And all this timestamp-checking is still covered in _get_or_head_response.","commit_id":"5b2c846c69cba6dc707ff32318b715791711f34e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2023,"context_line":"            # metadata headers for self.headers by selecting the source with"},{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_efc5e53d","line":2026,"range":{"start_line":2026,"start_character":46,"end_line":2026,"end_character":66},"updated":"2020-08-19 05:39:21.000000000","message":"This doesn\u0027t seem quite accurate anymore -- we may well have a timestamp for a bad_bucket now, yeah? Thinking of 404s from tombstones, in particular.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2023,"context_line":"            # metadata headers for self.headers by selecting the source with"},{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_566a73ec","line":2026,"range":{"start_line":2026,"start_character":46,"end_line":2026,"end_character":66},"in_reply_to":"9f560f44_efc5e53d","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2024,"context_line":"            # the latest X-Timestamp."},{"line_number":2025,"context_line":"            self.headers \u003d getter.last_headers"},{"line_number":2026,"context_line":"        elif (self.timestamp is not None and  # ie, not bad_bucket"},{"line_number":2027,"context_line":"              getter.last_headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027) !\u003d"},{"line_number":2028,"context_line":"              self.headers.get(\u0027X-Object-Sysmeta-Ec-Etag\u0027)):"},{"line_number":2029,"context_line":"            # Fragments at the same timestamp with different etags are never"},{"line_number":2030,"context_line":"            # expected. If somehow it happens then ignore those fragments"},{"line_number":2031,"context_line":"            # to avoid mixing fragments that will not reconstruct otherwise"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_afd00df9","line":2028,"range":{"start_line":2027,"start_character":14,"end_line":2028,"end_character":59},"updated":"2020-08-19 05:39:21.000000000","message":"...but then, none of the errors should ever have this meta, so this half should never be true. I guess it doesn\u0027t really matter?","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2062,"context_line":"    @property"},{"line_number":2063,"context_line":"    def shortfall(self):"},{"line_number":2064,"context_line":"        \"\"\""},{"line_number":2065,"context_line":"        The number of additional responses needed complete this bucket;"},{"line_number":2066,"context_line":"        typically (ndata - resp_count)."},{"line_number":2067,"context_line":""},{"line_number":2068,"context_line":"        If the bucket has no durable responses, shortfall is extended out to"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_8ff42989","line":2065,"range":{"start_line":2065,"start_character":43,"end_line":2065,"end_character":58},"updated":"2020-08-19 05:39:21.000000000","message":"nit:\n\n\u003e needed to complete","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2062,"context_line":"    @property"},{"line_number":2063,"context_line":"    def shortfall(self):"},{"line_number":2064,"context_line":"        \"\"\""},{"line_number":2065,"context_line":"        The number of additional responses needed complete this bucket;"},{"line_number":2066,"context_line":"        typically (ndata - resp_count)."},{"line_number":2067,"context_line":""},{"line_number":2068,"context_line":"        If the bucket has no durable responses, shortfall is extended out to"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_d1196d7a","line":2065,"range":{"start_line":2065,"start_character":43,"end_line":2065,"end_character":58},"in_reply_to":"9f560f44_8ff42989","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2081,"context_line":"        # for frags on the alt nodes."},{"line_number":2082,"context_line":"        alts \u003d set(self.alt_nodes.keys()).difference(set(self.gets.keys()))"},{"line_number":2083,"context_line":"        result \u003d self.policy.ec_ndata - (len(self.get_responses()) + len(alts))"},{"line_number":2084,"context_line":"        return max(result, 0)"},{"line_number":2085,"context_line":""},{"line_number":2086,"context_line":"    def __str__(self):"},{"line_number":2087,"context_line":"        # return a string summarising bucket state, useful for debugging."}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_6fb8b575","line":2084,"updated":"2020-08-19 05:39:21.000000000","message":"Seems weird that this calculation didn\u0027t need to change to complement shortfall()...","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2143,"context_line":"            self.add_bad_resp(get, parts_iter)"},{"line_number":2144,"context_line":""},{"line_number":2145,"context_line":"    def add_bad_resp(self, get, parts_iter):"},{"line_number":2146,"context_line":"        bad_bucket \u003d self._get_bad_bucket(get.last_status)"},{"line_number":2147,"context_line":"        bad_bucket.add_response(get, parts_iter)"},{"line_number":2148,"context_line":""},{"line_number":2149,"context_line":"    def add_good_response(self, get, parts_iter):"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_6a5a63f3","line":2146,"updated":"2020-08-19 05:39:21.000000000","message":"Should we have an early exit if get.last_status is None, like the old code did down in _get_or_head_response()?\n\nThe comment about\n\n # We may have spawned getters that find the node iterator\n # has been exhausted. Ignore them.\n # TODO: turns out that node_iter.nodes_left can bottom\n # out at \u003e0 when number of devs in ring is \u003c 2* replicas,\n # which definitely happens in tests and results in status\n # of None. We should fix that but keep this guard because\n # there is also a race between testing nodes_left/spawning\n # a getter and an existing getter calling next(node_iter).\n\nseemed significant...","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2143,"context_line":"            self.add_bad_resp(get, parts_iter)"},{"line_number":2144,"context_line":""},{"line_number":2145,"context_line":"    def add_bad_resp(self, get, parts_iter):"},{"line_number":2146,"context_line":"        bad_bucket \u003d self._get_bad_bucket(get.last_status)"},{"line_number":2147,"context_line":"        bad_bucket.add_response(get, parts_iter)"},{"line_number":2148,"context_line":""},{"line_number":2149,"context_line":"    def add_good_response(self, get, parts_iter):"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_117765a2","line":2146,"in_reply_to":"9f560f44_6a5a63f3","updated":"2020-08-21 19:03:09.000000000","message":"we can handle bad_bucket \u003d None, and the comment seemed to be indicating we had unnecessary complexity to handle a special case in tests; if that was necessary complexity that\u0027d be one thing - but tests seem fine and it\u0027s not obvious to me the code is incorrect as is.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c523bac62d7a1b3931acd437f3155bea87239c7","unresolved":false,"context_lines":[{"line_number":2651,"context_line":"            raise"},{"line_number":2652,"context_line":"        except ChunkWriteTimeout:"},{"line_number":2653,"context_line":"            self.app.logger.warning("},{"line_number":2654,"context_line":"                _(\u0027Client did not read from proxy within %ss\u0027) %"},{"line_number":2655,"context_line":"                self.app.client_timeout)"},{"line_number":2656,"context_line":"            self.app.logger.increment(\u0027client_timeouts\u0027)"},{"line_number":2657,"context_line":"        except GeneratorExit:"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_95f08f93","line":2654,"range":{"start_line":2654,"start_character":19,"end_line":2654,"end_character":60},"updated":"2020-08-20 23:20:50.000000000","message":"Wait, we\u0027re logging this once per fragment? That... doesn\u0027t seem right... :-/","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c523bac62d7a1b3931acd437f3155bea87239c7","unresolved":false,"context_lines":[{"line_number":2665,"context_line":"                        if end - begin + 1 \u003d\u003d self.bytes_used_from_backend:"},{"line_number":2666,"context_line":"                            warn \u003d False"},{"line_number":2667,"context_line":"            if not req.environ.get(\u0027swift.non_client_disconnect\u0027) and warn:"},{"line_number":2668,"context_line":"                self.app.logger.warning(\u0027Client disconnected on read of %r\u0027,"},{"line_number":2669,"context_line":"                                        self.path)"},{"line_number":2670,"context_line":"            raise"},{"line_number":2671,"context_line":"        except Exception:"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_5556f75d","line":2668,"range":{"start_line":2668,"start_character":41,"end_line":2668,"end_character":74},"updated":"2020-08-20 23:20:50.000000000","message":"This one, too.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2733,"context_line":"            self.status \u003d possible_source.status"},{"line_number":2734,"context_line":"            self.reason \u003d possible_source.reason"},{"line_number":2735,"context_line":"            self.body \u003d possible_source.read()"},{"line_number":2736,"context_line":"            self.source_headers \u003d possible_source.getheaders()"},{"line_number":2737,"context_line":""},{"line_number":2738,"context_line":"            if possible_source.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":2739,"context_line":"                self.app.error_limit(node, _(\u0027ERROR Insufficient Storage\u0027))"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_ea1b1318","line":2736,"updated":"2020-08-19 05:39:21.000000000","message":"The multiple returns are starting to get away from me, and the attr-setting feels repetitious. Maybe improved with http://paste.openstack.org/show/796930/ applied?","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2733,"context_line":"            self.status \u003d possible_source.status"},{"line_number":2734,"context_line":"            self.reason \u003d possible_source.reason"},{"line_number":2735,"context_line":"            self.body \u003d possible_source.read()"},{"line_number":2736,"context_line":"            self.source_headers \u003d possible_source.getheaders()"},{"line_number":2737,"context_line":""},{"line_number":2738,"context_line":"            if possible_source.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":2739,"context_line":"                self.app.error_limit(node, _(\u0027ERROR Insufficient Storage\u0027))"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_119c85be","line":2736,"in_reply_to":"9f560f44_ea1b1318","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c523bac62d7a1b3931acd437f3155bea87239c7","unresolved":false,"context_lines":[{"line_number":2767,"context_line":""},{"line_number":2768,"context_line":"    def _dig_for_source_and_node(self):"},{"line_number":2769,"context_line":"        for source, node in self.source_and_node_iter:"},{"line_number":2770,"context_line":"            if source and is_good_source(source.status):"},{"line_number":2771,"context_line":"                return source, node"},{"line_number":2772,"context_line":"        return None, None"},{"line_number":2773,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_d989a175","line":2770,"updated":"2020-08-20 23:20:50.000000000","message":"At this point, we\u0027re already yielding bytes out to the client; I\u0027m not sure the 416 carve-out makes sense here.\n\nThis also needs to pick up the old sense of used_source_etag, and use it when trying to determine whether this is a good source or not. That should resolve a lot of the trouble I hit in http://paste.openstack.org/show/797009/.\n\n(It was a pretty gnarly setup, unlikely to come up in practice, I suppose. Responses went\n\n* 404\n* etag1\n* 404\n* etag1\n* etag2\n* etag1\n* etag2\n* etag1\n* etag2\n* etag1\n* etag2\n* etag1\n* etag2\n* 404\n* etag2\n* 404\n\nwith a 1/4 chance of having an object-server stream blow up after 16MB. None of the 404s had timestamps, and all of the frags were durable.)\n\nSomething like http://paste.openstack.org/show/797010/ seems to square it.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2767,"context_line":""},{"line_number":2768,"context_line":"    def _dig_for_source_and_node(self):"},{"line_number":2769,"context_line":"        for source, node in self.source_and_node_iter:"},{"line_number":2770,"context_line":"            if source and is_good_source(source.status):"},{"line_number":2771,"context_line":"                return source, node"},{"line_number":2772,"context_line":"        return None, None"},{"line_number":2773,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_1185a55a","line":2770,"in_reply_to":"9f560f44_d989a175","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2845,"context_line":"            try:"},{"line_number":2846,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2847,"context_line":"            except Empty:"},{"line_number":2848,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2849,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2850,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2851,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_1066762a","line":2848,"range":{"start_line":2848,"start_character":29,"end_line":2848,"end_character":40},"updated":"2020-08-19 05:39:21.000000000","message":"Would it be worth doing something like\n\n with safe_iter.semaphore:\n     has_primaries_left \u003d bool(safe_iter.unsafe_iter.primaries_left)\n if has_primaries left:\n     ...\n\n? I\u0027d feel a bit better about it -- though given the description of the problem GreenthreadSafeIterator is trying to solve, maybe this read just *isn\u0027t an issue*...","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2845,"context_line":"            try:"},{"line_number":2846,"context_line":"                feeder_q.get(timeout\u003dself.app.concurrency_timeout)"},{"line_number":2847,"context_line":"            except Empty:"},{"line_number":2848,"context_line":"                if safe_iter.unsafe_iter.primaries_left:"},{"line_number":2849,"context_line":"                    # this will run async, if it ends up taking the last"},{"line_number":2850,"context_line":"                    # primary we won\u0027t find out until the next pass"},{"line_number":2851,"context_line":"                    pile.spawn(self._fragment_GET_request,"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_71da2178","line":2848,"range":{"start_line":2848,"start_character":29,"end_line":2848,"end_character":40},"in_reply_to":"9f560f44_1066762a","updated":"2020-08-21 19:03:09.000000000","message":"well, i don\u0027t that context is sufficiently wide enough to help if the worry is we spawn a _fragment_GET_request after someone already took the last primary.  As far as two threads trying to iterate the generator while it\u0027s blocked in execution - yeah that\u0027s not going to be an issue checking on the emptiness of the list.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2913,"context_line":"                    buckets.add_response(get, parts_iter)"},{"line_number":2914,"context_line":"                except ValueError as err:"},{"line_number":2915,"context_line":"                    self.app.logger.error("},{"line_number":2916,"context_line":"                        _(\"Problem with fragment response: %s\"), err)"},{"line_number":2917,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2918,"context_line":"                if best_bucket.durable and best_bucket.shortfall \u003c\u003d 0:"},{"line_number":2919,"context_line":"                    # good enough!"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_6aab8388","line":2916,"updated":"2020-08-19 05:39:21.000000000","message":"Can this catch anything *other* than the ec-etag mismatch error?","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2920,"context_line":"                    break"},{"line_number":2921,"context_line":"                requests_available \u003d extra_requests \u003c max_extra_requests and ("},{"line_number":2922,"context_line":"                    node_iter.nodes_left \u003e 0 or buckets.has_alternate_node())"},{"line_number":2923,"context_line":"                bad_resp \u003d not is_good_source(get.last_status)"},{"line_number":2924,"context_line":"                if requests_available and ("},{"line_number":2925,"context_line":"                        buckets.shortfall \u003e pile._pending or bad_resp):"},{"line_number":2926,"context_line":"                    extra_requests +\u003d 1"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_ea9e7361","line":2923,"updated":"2020-08-19 05:39:21.000000000","message":"So when resuming a GET, we might have a response that doesn\u0027t get added to *any* bucket, yet doesn\u0027t qualify as bad_resp, either... hmm...","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2922,"context_line":"                    node_iter.nodes_left \u003e 0 or buckets.has_alternate_node())"},{"line_number":2923,"context_line":"                bad_resp \u003d not is_good_source(get.last_status)"},{"line_number":2924,"context_line":"                if requests_available and ("},{"line_number":2925,"context_line":"                        buckets.shortfall \u003e pile._pending or bad_resp):"},{"line_number":2926,"context_line":"                    extra_requests +\u003d 1"},{"line_number":2927,"context_line":"                    pile.spawn(self._fragment_GET_request,"},{"line_number":2928,"context_line":"                               req, safe_iter, partition,"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_70b252e8","line":2925,"range":{"start_line":2925,"start_character":24,"end_line":2925,"end_character":57},"updated":"2020-08-19 05:39:21.000000000","message":"NB: this is the key comparison for why we wanted to rework the shortfall calculation -- it determines the level of concurrency we\u0027re willing to maintain as we continue deep into primaries (and even into handoffs). Previously, we\u0027d have a graph like\n\n C           |\n o           |\n n           |\n c     ndata |\\\n u           | \\\n r           |  \\\n r           |   \\\n e           |    \\\n n         1 |     ----------\n c           +-----------------------\n y               Responses in hand\n\n\nwhile now we\u0027ll have a graph more like\n\n C           |\n o           |\n n           |\n c     ndata |\\\n u           | \\\n r   nparity |  ------\n r           |        \\\n e           |         \\\n n         1 |          -----\n c           +-----------------------\n y               Responses in hand","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":2934,"context_line":"        # (but note that _fix_ranges() may also pop it back off before then)"},{"line_number":2935,"context_line":"        req.range \u003d orig_range"},{"line_number":2936,"context_line":"        best_bucket \u003d buckets.best_bucket"},{"line_number":2937,"context_line":"        if best_bucket and best_bucket.shortfall \u003c\u003d 0 and best_bucket.durable:"},{"line_number":2938,"context_line":"            # headers can come from any of the getters"},{"line_number":2939,"context_line":"            resp_headers \u003d best_bucket.headers"},{"line_number":2940,"context_line":"            resp_headers.pop(\u0027Content-Range\u0027, None)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_8f71c9e1","line":2937,"range":{"start_line":2937,"start_character":11,"end_line":2937,"end_character":22},"updated":"2020-08-19 05:39:21.000000000","message":"Always truthy, now. But at least the best_bucket.durable check should imply is_success(best_bucket.status)!","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":2934,"context_line":"        # (but note that _fix_ranges() may also pop it back off before then)"},{"line_number":2935,"context_line":"        req.range \u003d orig_range"},{"line_number":2936,"context_line":"        best_bucket \u003d buckets.best_bucket"},{"line_number":2937,"context_line":"        if best_bucket and best_bucket.shortfall \u003c\u003d 0 and best_bucket.durable:"},{"line_number":2938,"context_line":"            # headers can come from any of the getters"},{"line_number":2939,"context_line":"            resp_headers \u003d best_bucket.headers"},{"line_number":2940,"context_line":"            resp_headers.pop(\u0027Content-Range\u0027, None)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_b1b95916","line":2937,"range":{"start_line":2937,"start_character":11,"end_line":2937,"end_character":22},"in_reply_to":"9f560f44_8f71c9e1","updated":"2020-08-21 19:03:09.000000000","message":"Done","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":3001,"context_line":"                bodies.append(b\u0027\u0027)"},{"line_number":3002,"context_line":"                headers.append({})"},{"line_number":3003,"context_line":""},{"line_number":3004,"context_line":"            resp \u003d self.best_response("},{"line_number":3005,"context_line":"                req, statuses, reasons, bodies, \u0027Object\u0027,"},{"line_number":3006,"context_line":"                headers\u003dheaders)"},{"line_number":3007,"context_line":"        self._fix_response(req, resp)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_aa4abb23","line":3004,"updated":"2020-08-19 05:39:21.000000000","message":"I wonder if I\u0027d like the logging coming out of this call more if there was a\n\n elif is_success(best_bucket.status) and best_bucket.short_fall \u003e 0:\n     for getter, _ in best_bucket.get_responses():\n         statuses.append(getter.status)\n         ...\n\n... though I\u0027m not *sure* we\u0027d get the same responses :-/\n\nProbably better to just have ECObjectController override best_response and pull most of the logic here to the new function.\n\nw/e, all pre-existing cruft.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f5985c9cfd0c5e697ae5c304701335637036c90c","unresolved":false,"context_lines":[{"line_number":2790,"context_line":"            # old and new mid-stream and giving garbage to the client."},{"line_number":2791,"context_line":"            self.used_source_etag \u003d normalize_etag(src_headers.get("},{"line_number":2792,"context_line":"                \u0027x-object-sysmeta-ec-etag\u0027, src_headers.get(\u0027etag\u0027, \u0027\u0027)))"},{"line_number":2793,"context_line":"            self.node \u003d node"},{"line_number":2794,"context_line":"            return source, node"},{"line_number":2795,"context_line":"        return None, None"},{"line_number":2796,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_0828b894","side":"PARENT","line":2793,"updated":"2020-08-24 18:30:34.000000000","message":"so i guess I did drop this code on accident, glad we have a test to cover it!","commit_id":"5b2c846c69cba6dc707ff32318b715791711f34e"}],"test/unit/proxy/controllers/test_base.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1348,"context_line":"                count +\u003d 1"},{"line_number":1349,"context_line":"            self.assertEqual(count, 3)"},{"line_number":1350,"context_line":"            self.assertEqual(0, node_iter.primaries_left)"},{"line_number":1351,"context_line":"            # default fake_ring has NO handoffs, so nodes_left is kind of a lie"},{"line_number":1352,"context_line":"            self.assertEqual(3, node_iter.nodes_left)"},{"line_number":1353,"context_line":""},{"line_number":1354,"context_line":"    def test_iter_with_handoffs(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_cfb84812","line":1351,"updated":"2020-06-29 20:27:20.000000000","message":"Oh, interesting -- I hadn\u0027t really thought about it, but the same is true for our SAIOs. You exhaust node_iter with three primaries and a handoff, then it still claims to have 2 nodes_left...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":1348,"context_line":"                count +\u003d 1"},{"line_number":1349,"context_line":"            self.assertEqual(count, 3)"},{"line_number":1350,"context_line":"            self.assertEqual(0, node_iter.primaries_left)"},{"line_number":1351,"context_line":"            # default fake_ring has NO handoffs, so nodes_left is kind of a lie"},{"line_number":1352,"context_line":"            self.assertEqual(3, node_iter.nodes_left)"},{"line_number":1353,"context_line":""},{"line_number":1354,"context_line":"    def test_iter_with_handoffs(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_642caf4a","line":1351,"in_reply_to":"bf51134e_351787a6","updated":"2020-06-30 18:58:51.000000000","message":"I mean, we could cap it at len(ring.devices) -- but IDK what it\u0027d really buy us. I feel like it\u0027ll just lead to some other kind of confusion somewhere else.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1348,"context_line":"                count +\u003d 1"},{"line_number":1349,"context_line":"            self.assertEqual(count, 3)"},{"line_number":1350,"context_line":"            self.assertEqual(0, node_iter.primaries_left)"},{"line_number":1351,"context_line":"            # default fake_ring has NO handoffs, so nodes_left is kind of a lie"},{"line_number":1352,"context_line":"            self.assertEqual(3, node_iter.nodes_left)"},{"line_number":1353,"context_line":""},{"line_number":1354,"context_line":"    def test_iter_with_handoffs(self):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_351787a6","line":1351,"in_reply_to":"bf51134e_cfb84812","updated":"2020-06-30 14:43:40.000000000","message":"yeah, it\u0027s not obvious to me how to make it better","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1352,"context_line":"            self.assertEqual(3, node_iter.nodes_left)"},{"line_number":1353,"context_line":""},{"line_number":1354,"context_line":"    def test_iter_with_handoffs(self):"},{"line_number":1355,"context_line":"        ring \u003d FakeRing(replicas\u003d3, max_more_nodes\u003d20)  # handoffs avialable"},{"line_number":1356,"context_line":"        policy \u003d StoragePolicy(0, \u0027zero\u0027, object_ring\u003dring)"},{"line_number":1357,"context_line":"        node_iter \u003d NodeIter(self.app, policy.object_ring, 0, policy\u003dpolicy)"},{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_2f3904a3","line":1355,"range":{"start_line":1355,"start_character":67,"end_line":1355,"end_character":76},"updated":"2020-06-29 20:27:20.000000000","message":"nit: available","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1352,"context_line":"            self.assertEqual(3, node_iter.nodes_left)"},{"line_number":1353,"context_line":""},{"line_number":1354,"context_line":"    def test_iter_with_handoffs(self):"},{"line_number":1355,"context_line":"        ring \u003d FakeRing(replicas\u003d3, max_more_nodes\u003d20)  # handoffs avialable"},{"line_number":1356,"context_line":"        policy \u003d StoragePolicy(0, \u0027zero\u0027, object_ring\u003dring)"},{"line_number":1357,"context_line":"        node_iter \u003d NodeIter(self.app, policy.object_ring, 0, policy\u003dpolicy)"},{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f5434fae","line":1355,"range":{"start_line":1355,"start_character":67,"end_line":1355,"end_character":76},"in_reply_to":"bf51134e_2f3904a3","updated":"2020-06-30 14:43:40.000000000","message":"#willfix","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"},{"line_number":1359,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1360,"context_line":"        primary_indexes \u003d set()"},{"line_number":1361,"context_line":"        handoff_indexes \u003d set()"},{"line_number":1362,"context_line":"        count \u003d 0"},{"line_number":1363,"context_line":"        for node in node_iter:"},{"line_number":1364,"context_line":"            if \u0027index\u0027 in node:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_ef526cdd","line":1361,"updated":"2020-06-29 20:27:20.000000000","message":"Stronger test if we track these in a list -- they should always come out in the same order.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f48b29521db73750daa4588dcebbe033c2e452be","unresolved":false,"context_lines":[{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"},{"line_number":1359,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1360,"context_line":"        primary_indexes \u003d set()"},{"line_number":1361,"context_line":"        handoff_indexes \u003d set()"},{"line_number":1362,"context_line":"        count \u003d 0"},{"line_number":1363,"context_line":"        for node in node_iter:"},{"line_number":1364,"context_line":"            if \u0027index\u0027 in node:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_64030fc0","line":1361,"in_reply_to":"bf51134e_152083f7","updated":"2020-06-30 18:58:51.000000000","message":"I just meant for the handoffs -- primaries presumably will get shuffled.","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cd572cdac058ab1dd82acc2745af0b4d9cd0c3f8","unresolved":false,"context_lines":[{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"},{"line_number":1359,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1360,"context_line":"        primary_indexes \u003d set()"},{"line_number":1361,"context_line":"        handoff_indexes \u003d set()"},{"line_number":1362,"context_line":"        count \u003d 0"},{"line_number":1363,"context_line":"        for node in node_iter:"},{"line_number":1364,"context_line":"            if \u0027index\u0027 in node:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_ead7e94e","line":1361,"in_reply_to":"bf51134e_64030fc0","updated":"2020-06-30 19:38:41.000000000","message":"OH!  dur :D","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1358,"context_line":"        self.assertEqual(6, node_iter.nodes_left)"},{"line_number":1359,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1360,"context_line":"        primary_indexes \u003d set()"},{"line_number":1361,"context_line":"        handoff_indexes \u003d set()"},{"line_number":1362,"context_line":"        count \u003d 0"},{"line_number":1363,"context_line":"        for node in node_iter:"},{"line_number":1364,"context_line":"            if \u0027index\u0027 in node:"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_152083f7","line":1361,"in_reply_to":"bf51134e_ef526cdd","updated":"2020-06-30 14:43:40.000000000","message":"really?  there\u0027s no shuffling?  I feel like node order is always getting shuffled.  I\u0027ll try it...","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1387,"context_line":""},{"line_number":1388,"context_line":"        def eat_node(node_iter):"},{"line_number":1389,"context_line":"            for node in node_iter:"},{"line_number":1390,"context_line":"                return node"},{"line_number":1391,"context_line":""},{"line_number":1392,"context_line":"        safe_iter \u003d GreenthreadSafeIterator(node_iter)"},{"line_number":1393,"context_line":"        for i in range(5):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_2fbe24d4","line":1390,"updated":"2020-06-29 20:27:20.000000000","message":"Not just\n\n return next(node_iter)\n\n?","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1387,"context_line":""},{"line_number":1388,"context_line":"        def eat_node(node_iter):"},{"line_number":1389,"context_line":"            for node in node_iter:"},{"line_number":1390,"context_line":"                return node"},{"line_number":1391,"context_line":""},{"line_number":1392,"context_line":"        safe_iter \u003d GreenthreadSafeIterator(node_iter)"},{"line_number":1393,"context_line":"        for i in range(5):"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_7538df3c","line":1390,"in_reply_to":"bf51134e_2fbe24d4","updated":"2020-06-30 14:43:40.000000000","message":"oh yeah, that\u0027d be more better","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c54a297bb08e97940441cafdeed3cb73bae9816","unresolved":false,"context_lines":[{"line_number":1401,"context_line":"        self.assertEqual(5, len(primary_indexes))"},{"line_number":1402,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1403,"context_line":""},{"line_number":1404,"context_line":"        # it\u0027s problematic we don\u0027t decrement nodes_left until we resume"},{"line_number":1405,"context_line":"        self.assertEqual(12, node_iter.nodes_left)"},{"line_number":1406,"context_line":"        for node in node_iter:"},{"line_number":1407,"context_line":"            nodes.append(node)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_cfc20853","line":1404,"updated":"2020-06-29 20:27:20.000000000","message":"Hmm... I suppose it\u0027s because we check to see whether the node got error limited *just now* as part of the request? IDK that I like that behavior, though -- with the concurrent requests for EC, we may well be pulling ndata nodes at a time, well before we\u0027d expect to have a settled response to use to error limit. test_iter_nodes_gives_extra_if_error_limited_inline feels very weird to me -- I don\u0027t understand why we started doing that back in https://review.opendev.org/#/c/26290/ :-/","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0fa667f0f35bb4436376749659116db38e40391d","unresolved":false,"context_lines":[{"line_number":1401,"context_line":"        self.assertEqual(5, len(primary_indexes))"},{"line_number":1402,"context_line":"        self.assertEqual(3, node_iter.primaries_left)"},{"line_number":1403,"context_line":""},{"line_number":1404,"context_line":"        # it\u0027s problematic we don\u0027t decrement nodes_left until we resume"},{"line_number":1405,"context_line":"        self.assertEqual(12, node_iter.nodes_left)"},{"line_number":1406,"context_line":"        for node in node_iter:"},{"line_number":1407,"context_line":"            nodes.append(node)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_f5eeafb2","line":1404,"in_reply_to":"bf51134e_cfc20853","updated":"2020-06-30 14:43:40.000000000","message":"yeah some of these tests were just to codify all the crazy behaviors I saw so we can talk about \u0027em","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":2728,"context_line":"            obj2[\u0027etag\u0027]: self.policy.ec_ndata,"},{"line_number":2729,"context_line":"        }"},{"line_number":2730,"context_line":"        self.assertEqual(expected, {"},{"line_number":2731,"context_line":"            e: len(f) for e, f in collected_responses.items()})"},{"line_number":2732,"context_line":""},{"line_number":2733,"context_line":"    def test_GET_with_many_missed_overwrite_will_need_handoff(self):"},{"line_number":2734,"context_line":"        obj1 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj1\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_32d3793a","line":2731,"updated":"2020-06-02 17:27:36.000000000","message":"I removed shuffle on this test; because I think it makes the test *stronger* - but I suppose there must have been another case where obj2[\u0027etag\u0027]: self.policy.ec_ndata + 1 which is now missing if that\u0027s significant.","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":3814,"context_line":"        log_msg_args, log_msg_kwargs \u003d self.logger.log_dict[\u0027error\u0027][0]"},{"line_number":3815,"context_line":"        self.assertEqual(log_msg_kwargs[\u0027exc_info\u0027][0], ECDriverError)"},{"line_number":3816,"context_line":""},{"line_number":3817,"context_line":"    def test_GET_read_timeout(self):"},{"line_number":3818,"context_line":"        segment_size \u003d self.policy.ec_segment_size"},{"line_number":3819,"context_line":"        test_data \u003d (b\u0027test\u0027 * segment_size)[:-333]"},{"line_number":3820,"context_line":"        etag \u003d md5(test_data).hexdigest()"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_32dd3914","line":3817,"updated":"2020-06-02 17:27:36.000000000","message":"This is the base case for how we fail when we\u0027re reading off the backend and ALL our responses get slow with no good to resume from.","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":3840,"context_line":"            self.assertNotEqual(md5(resp.body).hexdigest(), etag)"},{"line_number":3841,"context_line":"        error_lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":3842,"context_line":"        nparity \u003d self.policy.ec_nparity"},{"line_number":3843,"context_line":"        self.assertGreater(len(error_lines), nparity)"},{"line_number":3844,"context_line":"        for line in error_lines[:nparity]:"},{"line_number":3845,"context_line":"            self.assertIn(\u0027retrying\u0027, line)"},{"line_number":3846,"context_line":"        for line in error_lines[nparity:]:"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_72ae9181","line":3843,"updated":"2020-06-02 17:27:36.000000000","message":"this one was tricky, sometimes the frag fetching gt\u0027s would get killed (instead of timeout) while the mainthread was collecting off the queue and one of the iters gave back a None\n\nI\u0027m not sure if the old code managed to get 1:1 timeouts to fragfetcher because of lucky eventlet timing voodoo or because somehow the old resume code was more robust.","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b12d82852810b3ca758e74ff4c38519557c651a9","unresolved":false,"context_lines":[{"line_number":3846,"context_line":"        for line in error_lines[nparity:]:"},{"line_number":3847,"context_line":"            self.assertIn(\u0027ChunkReadTimeout (0.01s)\u0027, line)"},{"line_number":3848,"context_line":""},{"line_number":3849,"context_line":"    def test_GET_read_timeout_resume(self):"},{"line_number":3850,"context_line":"        segment_size \u003d self.policy.ec_segment_size"},{"line_number":3851,"context_line":"        test_data \u003d (b\u0027test\u0027 * segment_size)[:-333]"},{"line_number":3852,"context_line":"        etag \u003d md5(test_data).hexdigest()"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_72fcd171","line":3849,"updated":"2020-06-02 17:27:36.000000000","message":"This test demonstrates ec download resume when node read times out.  As part of this work I\u0027d like to add more variety of backend failure scenarios to round out our understanding of the behaviors we want.","commit_id":"e86890ff96280b6eacd30300a1272bc59d76f489"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2417,"context_line":"            self.app, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027)"},{"line_number":2418,"context_line":"        safe_iter \u003d mock.MagicMock()"},{"line_number":2419,"context_line":"        safe_iter.unsafe_iter.primary_nodes.__nonzero__.side_effect \u003d ["},{"line_number":2420,"context_line":"            True, True, False]"},{"line_number":2421,"context_line":"        pile \u003d mock.MagicMock()"},{"line_number":2422,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":2423,"context_line":"        buckets \u003d mock.MagicMock()"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_265f31e3","line":2420,"updated":"2020-06-17 13:13:52.000000000","message":"maybe i\u0027ll try again to use a real wrapped node iter","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2417,"context_line":"            self.app, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027)"},{"line_number":2418,"context_line":"        safe_iter \u003d mock.MagicMock()"},{"line_number":2419,"context_line":"        safe_iter.unsafe_iter.primary_nodes.__nonzero__.side_effect \u003d ["},{"line_number":2420,"context_line":"            True, True, False]"},{"line_number":2421,"context_line":"        pile \u003d mock.MagicMock()"},{"line_number":2422,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":2423,"context_line":"        buckets \u003d mock.MagicMock()"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_62844ed5","line":2420,"in_reply_to":"bf51134e_265f31e3","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2421,"context_line":"        pile \u003d mock.MagicMock()"},{"line_number":2422,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":2423,"context_line":"        buckets \u003d mock.MagicMock()"},{"line_number":2424,"context_line":"        feeder_q \u003d mock.MagicMock()"},{"line_number":2425,"context_line":"        feeder_q.get.side_effect \u003d Empty"},{"line_number":2426,"context_line":"        controller.feed_remaining_primaries("},{"line_number":2427,"context_line":"            safe_iter, pile, req, 0, self.policy, buckets, feeder_q)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_66c2c987","line":2424,"updated":"2020-06-17 13:13:52.000000000","message":"this test is a nightmare of mocks","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f66588e8b8d8fde22097938062cb6ec3b8b97290","unresolved":false,"context_lines":[{"line_number":2421,"context_line":"        pile \u003d mock.MagicMock()"},{"line_number":2422,"context_line":"        req \u003d swob.Request.blank(\u0027/v1/a/c/o\u0027)"},{"line_number":2423,"context_line":"        buckets \u003d mock.MagicMock()"},{"line_number":2424,"context_line":"        feeder_q \u003d mock.MagicMock()"},{"line_number":2425,"context_line":"        feeder_q.get.side_effect \u003d Empty"},{"line_number":2426,"context_line":"        controller.feed_remaining_primaries("},{"line_number":2427,"context_line":"            safe_iter, pile, req, 0, self.policy, buckets, feeder_q)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_42878ad8","line":2424,"in_reply_to":"bf51134e_66c2c987","updated":"2020-06-18 21:58:08.000000000","message":"Done","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":2429,"context_line":"                             0, self.policy, buckets.get_extra_headers)"},{"line_number":2430,"context_line":"        self.assertEqual(pile.spawn.call_args_list, [expected] * 2)"},{"line_number":2431,"context_line":"        self.assertEqual(feeder_q.get.call_args_list, ["},{"line_number":2432,"context_line":"            mock.call(timeout\u003dself.app.concurrency_timeout)] * 3)"},{"line_number":2433,"context_line":""},{"line_number":2434,"context_line":"    def test_GET_timeout(self):"},{"line_number":2435,"context_line":"        req \u003d swift.common.swob.Request.blank(\u0027/v1/a/c/o\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_66700974","line":2432,"updated":"2020-06-17 13:13:52.000000000","message":"but when i add support for per-policy/configurable timeouts this assertion will be important","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"de30bf7e1ced51663e970d6ee83f069a255056cf","unresolved":false,"context_lines":[{"line_number":3243,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 11, \u0027durable\u0027: False},  # parity"},{"line_number":3244,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False},  # parity"},{"line_number":3245,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},  # parity"},{"line_number":3246,"context_line":"        ] + [[]] * self.replicas()  # handoffs all 404"},{"line_number":3247,"context_line":""},{"line_number":3248,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3249,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"bf51134e_e63259a7","line":3246,"updated":"2020-06-17 13:13:52.000000000","message":"this is related to the multi-bucket rate","commit_id":"bde7df7f7e9208d22b1c4481cb80e98d7410373e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"151f9e148e9f4f818afceeb41d5b146a07465384","unresolved":false,"context_lines":[{"line_number":4038,"context_line":"        self.assertEqual(resp.etag, body_etag)"},{"line_number":4039,"context_line":"        self.assertEqual(resp.headers[\u0027Accept-Ranges\u0027], \u0027bytes\u0027)"},{"line_number":4040,"context_line":""},{"line_number":4041,"context_line":"    def test_non_durable_ec_response_bucket(self):"},{"line_number":4042,"context_line":"        ts \u003d self.ts()"},{"line_number":4043,"context_line":"        bucket \u003d obj.ECGetResponseBucket(self.policy, ts.internal)"},{"line_number":4044,"context_line":"        self.assertEqual(bucket.shortfall, self.policy.ec_ndata)"}],"source_content_type":"text/x-python","patch_set":12,"id":"bf51134e_b745cd45","line":4041,"updated":"2020-06-22 22:45:11.000000000","message":"mostly to support the inscrutable code at https://review.opendev.org/#/c/711342/12/swift/proxy/controllers/obj.py@2063","commit_id":"2c02a67114595b49fa335fedb17c0aa518753f2f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e821d3b6abd579487844d71de3660d6f2d4dea02","unresolved":false,"context_lines":[{"line_number":4135,"context_line":"        self.assertEqual(resp.etag, body_etag)"},{"line_number":4136,"context_line":"        self.assertEqual(resp.headers[\u0027Accept-Ranges\u0027], \u0027bytes\u0027)"},{"line_number":4137,"context_line":""},{"line_number":4138,"context_line":"    def test_non_durable_ec_response_bucket(self):"},{"line_number":4139,"context_line":"        ts \u003d self.ts()"},{"line_number":4140,"context_line":"        bucket \u003d obj.ECGetResponseBucket(self.policy, ts)"},{"line_number":4141,"context_line":"        self.assertEqual(bucket.shortfall, self.policy.ec_ndata)"}],"source_content_type":"text/x-python","patch_set":18,"id":"9f560f44_4fa2114d","line":4138,"updated":"2020-08-19 05:39:21.000000000","message":"Should we do a comparable test for a duplicated policy? I suppose it\u0027d look much the same, but with a *whole bunch* of extra nparity shortfalls...\n\nThat seems a little weird to me, but I can\u0027t quite put my finger on *why*.","commit_id":"8839e7b3ead0a49189aa4e0ba40e05ffd459cf5e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f5985c9cfd0c5e697ae5c304701335637036c90c","unresolved":false,"context_lines":[{"line_number":4063,"context_line":"        headers2 \u003d {\u0027X-Object-Sysmeta-Ec-Etag\u0027: etag2,"},{"line_number":4064,"context_line":"                    \u0027X-Backend-Timestamp\u0027: self.ts().internal}"},{"line_number":4065,"context_line":"        headers1 \u003d {\u0027X-Object-Sysmeta-Ec-Etag\u0027: etag1,"},{"line_number":4066,"context_line":"                    \u0027X-Backend-Timestamp\u0027: self.ts().internal}"},{"line_number":4067,"context_line":"        responses \u003d ["},{"line_number":4068,"context_line":"            # 404"},{"line_number":4069,"context_line":"            (404, [b\u0027\u0027], {}),"}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_c30fc32b","line":4066,"updated":"2020-08-24 18:30:34.000000000","message":"since these didn\u0027t include the content length header, the range iteration thought they were zero byte responses and threw away the bodies - yuk","commit_id":"1394bbd3134c22207437e73d9c727c7320dde645"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"83c52ba35a1a84a456b2990a5324180720d09fe9","unresolved":false,"context_lines":[{"line_number":4118,"context_line":"            # 404"},{"line_number":4119,"context_line":"            (404, [b\u0027\u0027], {}),"},{"line_number":4120,"context_line":"            # etag2"},{"line_number":4121,"context_line":"            (200, ec_archive_bodies2[3], self._add_frag_index(3, headers2)),"},{"line_number":4122,"context_line":"            # 404"},{"line_number":4123,"context_line":"            (404, [b\u0027\u0027], {}),"},{"line_number":4124,"context_line":"            # etag1"}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_038f1acd","line":4121,"updated":"2020-08-21 19:03:09.000000000","message":"I\u0027d be surprised if master was handling this correctly - but I like that adding concurrent gets to ec is actually making the code and behaviors more robust and better tested!  That\u0027s a good sign we\u0027re getting close!","commit_id":"1394bbd3134c22207437e73d9c727c7320dde645"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"243d30dd5b091b54e63a6f73f36eb0e28371a701","unresolved":false,"context_lines":[{"line_number":4118,"context_line":"            # 404"},{"line_number":4119,"context_line":"            (404, [b\u0027\u0027], {}),"},{"line_number":4120,"context_line":"            # etag2"},{"line_number":4121,"context_line":"            (200, ec_archive_bodies2[3], self._add_frag_index(3, headers2)),"},{"line_number":4122,"context_line":"            # 404"},{"line_number":4123,"context_line":"            (404, [b\u0027\u0027], {}),"},{"line_number":4124,"context_line":"            # etag1"}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_ac0a9b22","line":4121,"in_reply_to":"9f560f44_038f1acd","updated":"2020-08-21 23:39:06.000000000","message":"FWIW, this test still passes after a\n\n git checkout @~ -- swift\n\nThat Sam\u0027s a smart guy, I bet he made sure it worked right! ;-)","commit_id":"1394bbd3134c22207437e73d9c727c7320dde645"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"243d30dd5b091b54e63a6f73f36eb0e28371a701","unresolved":false,"context_lines":[{"line_number":4133,"context_line":"                              headers\u003dheaders):"},{"line_number":4134,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":4135,"context_line":"            self.assertEqual(resp.status_int, 200)"},{"line_number":4136,"context_line":"            self.assertTrue(md5(resp.body).hexdigest(), etag1)"},{"line_number":4137,"context_line":"        error_lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":4138,"context_line":"        self.assertEqual(2, len(error_lines))"},{"line_number":4139,"context_line":"        for line in error_lines:"}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_66a0a443","line":4136,"range":{"start_line":4136,"start_character":17,"end_line":4136,"end_character":27},"updated":"2020-08-21 23:39:06.000000000","message":"assertEqual, yeah? Oh, I see, it got copied from test_GET_read_timeout_resume ...\n\nWhen I fix them, looks like resp.body must be empty :-(\n\nFAIL: test_GET_read_timeout_resume_mixed_etag (test.unit.proxy.controllers.test_obj.TestECObjController)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/vagrant/swift/test/unit/proxy/controllers/test_obj.py\", line 4136, in test_GET_read_timeout_resume_mixed_etag\n    self.assertEqual(md5(resp.body).hexdigest(), etag1)\nAssertionError: \u0027d41d8cd98f00b204e9800998ecf8427e\u0027 !\u003d \u00273f3f757978b31b67f84883e5b15122bd\u0027\n- d41d8cd98f00b204e9800998ecf8427e\n+ 3f3f757978b31b67f84883e5b15122bd","commit_id":"1394bbd3134c22207437e73d9c727c7320dde645"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f5985c9cfd0c5e697ae5c304701335637036c90c","unresolved":false,"context_lines":[{"line_number":4133,"context_line":"                              headers\u003dheaders):"},{"line_number":4134,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":4135,"context_line":"            self.assertEqual(resp.status_int, 200)"},{"line_number":4136,"context_line":"            self.assertTrue(md5(resp.body).hexdigest(), etag1)"},{"line_number":4137,"context_line":"        error_lines \u003d self.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":4138,"context_line":"        self.assertEqual(2, len(error_lines))"},{"line_number":4139,"context_line":"        for line in error_lines:"}],"source_content_type":"text/x-python","patch_set":19,"id":"9f560f44_a3056f12","line":4136,"range":{"start_line":4136,"start_character":17,"end_line":4136,"end_character":27},"in_reply_to":"9f560f44_66a0a443","updated":"2020-08-24 18:30:34.000000000","message":"ugh, that\u0027s a *really* frustrating mistake!\n\nLuckily the same fix seems to apply to both","commit_id":"1394bbd3134c22207437e73d9c727c7320dde645"}]}