)]}'
{"etc/proxy-server.conf-sample":[{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"c0202ae4c65bcba778a342a0edf45f97672273aa","unresolved":false,"context_lines":[{"line_number":141,"context_line":"# would be more likely to cause 503 Service Unavailable responses to object"},{"line_number":142,"context_line":"# reads. Set this to true to return to the prior behavior of considering such"},{"line_number":143,"context_line":"# 404s when establishing quorum."},{"line_number":144,"context_line":"# consider_404s_from_object_handoffs \u003d False"},{"line_number":145,"context_line":"#"},{"line_number":146,"context_line":"# object_chunk_size \u003d 65536"},{"line_number":147,"context_line":"# client_chunk_size \u003d 65536"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"7faddb67_1956ff2b","line":144,"updated":"2019-07-30 10:50:29.000000000","message":"I\u0027m not a fan of adding links to bugs in code, but I\u0027d make an exception here and add the link to [1]. I would also add a note about this option being \"temporary\" and how we plan on removing after \u0027N\u0027 releases. We want to move everyone to the new \"correct\" world.\n\n[1] - https://bugs.launchpad.net/swift/+bug/1837819","commit_id":"b29d43397e3ed8ada1c7ffa16120faf193675db6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b74ed7a5c51c8168d4a15f4fb1c09915e6588175","unresolved":false,"context_lines":[{"line_number":141,"context_line":"# would be more likely to cause 503 Service Unavailable responses to object"},{"line_number":142,"context_line":"# reads. Set this to true to return to the prior behavior of considering such"},{"line_number":143,"context_line":"# 404s when establishing quorum."},{"line_number":144,"context_line":"# consider_404s_from_object_handoffs \u003d False"},{"line_number":145,"context_line":"#"},{"line_number":146,"context_line":"# object_chunk_size \u003d 65536"},{"line_number":147,"context_line":"# client_chunk_size \u003d 65536"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"7faddb67_9f696ff9","line":144,"in_reply_to":"7faddb67_1956ff2b","updated":"2019-07-30 13:02:39.000000000","message":"Generally I\u0027m a big fan of putting power in the hands of the operator, but this one feels like a really hard sell!\n\nEither we expect one of the primary nodes to return a response or we don\u0027t ... and if we *don\u0027t* then what are we even doing.  I\u0027m not oblivious to the fact that service quality metrics might be related to # of 5XX responses, but I see no good reason the client response shouldn\u0027t indicate the difference between \"not found\" and \"unable to contact primaries\" - we should tell the truth.\n\nPerhaps the config option is:\n\n\"return server errors as client errors\"\n\n... and then why limit it to just this one case that shows up under the duress of extreme load.","commit_id":"b29d43397e3ed8ada1c7ffa16120faf193675db6"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":1285,"context_line":"            if \u0027handoff_index\u0027 in node and \\"},{"line_number":1286,"context_line":"                    possible_source.status \u003d\u003d HTTP_NOT_FOUND and \\"},{"line_number":1287,"context_line":"                    not Timestamp(src_headers.get(\u0027x-backend-timestamp\u0027, 0)):"},{"line_number":1288,"context_line":"                # throw out 404s from handoff nodes unless the data is really"},{"line_number":1289,"context_line":"                # on disk and had been DELETEd"},{"line_number":1290,"context_line":"                return False"},{"line_number":1291,"context_line":"            self.statuses.append(possible_source.status)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_9390ea50","line":1288,"updated":"2019-07-23 17:36:39.000000000","message":"this seems like a solid move toward a better model for thinking about response information","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b74ed7a5c51c8168d4a15f4fb1c09915e6588175","unresolved":false,"context_lines":[{"line_number":1285,"context_line":"            should_consider_404s_from_handoffs \u003d False"},{"line_number":1286,"context_line":"            if self.server_type \u003d\u003d \u0027Object\u0027:"},{"line_number":1287,"context_line":"                should_consider_404s_from_handoffs \u003d \\"},{"line_number":1288,"context_line":"                    self.app.consider_404s_from_object_handoffs"},{"line_number":1289,"context_line":""},{"line_number":1290,"context_line":"            if not should_consider_404s_from_handoffs and \\"},{"line_number":1291,"context_line":"                    \u0027handoff_index\u0027 in node and \\"}],"source_content_type":"text/x-python","patch_set":7,"id":"7faddb67_ca0ac31e","line":1288,"updated":"2019-07-30 13:02:39.000000000","message":"oh dear\n\nI couldn\u0027t find any unit test failures in proxy.controller.test_obj if just pin this to True/False?","commit_id":"b29d43397e3ed8ada1c7ffa16120faf193675db6"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2427,"context_line":"                # catch any HTTPException response here so that we can"},{"line_number":2428,"context_line":"                # process response headers uniformly in _fix_response"},{"line_number":2429,"context_line":"                resp \u003d err_resp"},{"line_number":2430,"context_line":"        else:"},{"line_number":2431,"context_line":"            # TODO: we can get here if all buckets are successful but none"},{"line_number":2432,"context_line":"            # have ec_ndata getters, so bad_bucket may have no gets and we will"},{"line_number":2433,"context_line":"            # return a 503 when a 404 may be more appropriate. We can also get"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_7320ce4a","line":2430,"updated":"2019-07-23 17:36:39.000000000","message":"so given that we\u0027re not going to be returning from a *good* bucket...","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2439,"context_line":"            bodies \u003d []"},{"line_number":2440,"context_line":"            headers \u003d []"},{"line_number":2441,"context_line":"            for getter, _parts_iter in bad_bucket.get_responses():"},{"line_number":2442,"context_line":"                if best_bucket and best_bucket.durable:"},{"line_number":2443,"context_line":"                    headers \u003d HeaderKeyDict(getter.last_headers)"},{"line_number":2444,"context_line":"                    t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":2445,"context_line":"                    t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_931d4a11","line":2442,"updated":"2019-07-23 17:36:39.000000000","message":"but we did *have* a \"good\" bucket, it just wasn\u0027t \"good enough\"","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2445,"context_line":"                    t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"},{"line_number":2446,"context_line":"                                        headers.get(\u0027X-Timestamp\u0027))"},{"line_number":2447,"context_line":"                    bad_ts \u003d Timestamp(t_data_file or t_obj or \u00270\u0027)"},{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_5317d2ee","line":2448,"updated":"2019-07-23 17:36:39.000000000","message":"then we can say \"if *this* bad bucket is older than our \"not good enough bucket\"","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"},{"line_number":2452,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2453,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2454,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_b3dba64d","line":2451,"updated":"2019-07-23 17:36:39.000000000","message":"skip it, and look for some other like... \"better\" bad bucket?","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"},{"line_number":2452,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2453,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2454,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_c95a1882","line":2451,"in_reply_to":"7faddb67_7e6895b9","updated":"2019-07-25 01:29:39.000000000","message":"I\u0027m not so sure that ECGetResponseBucket is actually well suited for this, though :-/\n\nI\u0027ve been seeing some \u0027Etag mismatch\u0027 log lines, and I *think* it\u0027s because this one bad_bucket mixes 404s (which don\u0027t have etags) with 416s (which *do*).\n\nMaybe I could resolve that somewhat by looking at self.timestamp_str to decide whether to do the etag check? Feels hacky.\n\nI\u0027m also weirded out by the fact that none of\n\n- 404 with a timestamp,\n- 304,\n- 412, or\n- 416\n\nis considered \"good\"...","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be1bbf1e3b482153bfa563dcd8ed61e1109cc991","unresolved":false,"context_lines":[{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"},{"line_number":2452,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2453,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2454,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_7e6895b9","line":2451,"in_reply_to":"7faddb67_b35ac680","updated":"2019-07-23 18:05:54.000000000","message":"oic, bad_bucket.get_responses is just all the bad responses - thank you for clarifying my misconception","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"},{"line_number":2452,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2453,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2454,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_b35ac680","line":2451,"in_reply_to":"7faddb67_b3dba64d","updated":"2019-07-23 17:59:25.000000000","message":"Technically, it\u0027s all the same bad bucket, but yeah. The idea is that we ought to go digging through the bad in case one of them knows about a newer tombstone which would prompt us to 404.","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b9f26a36ff9919b4771368d634531d212afd40ba","unresolved":false,"context_lines":[{"line_number":2448,"context_line":"                    if bad_ts \u003c\u003d Timestamp(best_bucket.timestamp_str):"},{"line_number":2449,"context_line":"                        # We have reason to believe there\u0027s still good data"},{"line_number":2450,"context_line":"                        # out there, it\u0027s just currently unavailable"},{"line_number":2451,"context_line":"                        continue"},{"line_number":2452,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2453,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2454,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_7883a605","line":2451,"in_reply_to":"7faddb67_c95a1882","updated":"2019-07-25 15:39:01.000000000","message":"Did the hack, if for no other reason than that I *really* want that etag mismatch to be a GIANT red flag and false positives diminish that.\n\nNothing broke, which means that I probably ought to be adding more tests...","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2455,"context_line":"                headers.extend(getter.source_headers)"},{"line_number":2456,"context_line":"            resp \u003d self.best_response("},{"line_number":2457,"context_line":"                req, statuses, reasons, bodies, \u0027Object\u0027,"},{"line_number":2458,"context_line":"                headers\u003dheaders)"},{"line_number":2459,"context_line":"        self._fix_response(req, resp)"},{"line_number":2460,"context_line":"        return resp"},{"line_number":2461,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_53ecb2f2","line":2458,"updated":"2019-07-23 17:36:39.000000000","message":"And if none of the bad buckets are \"good enough\" we end up cooking up a response on no information at all.... hrmm, not obvious that\u0027s the best way to proceed but seems likely to be better than using sort of \"known out of date\" information","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2387,"context_line":"                best_bucket \u003d buckets.best_bucket"},{"line_number":2388,"context_line":"                if best_bucket:"},{"line_number":2389,"context_line":"                    shortfall \u003d best_bucket.shortfall"},{"line_number":2390,"context_line":"                    if not best_bucket.durable and shortfall \u003c\u003d 0:"},{"line_number":2391,"context_line":"                        # be willing to go a *little* deeper, slowly"},{"line_number":2392,"context_line":"                        shortfall \u003d 1"},{"line_number":2393,"context_line":"                    shortfall \u003d min(shortfall, bad_bucket.shortfall)"},{"line_number":2394,"context_line":"                if (extra_requests \u003c max_extra_requests and"},{"line_number":2395,"context_line":"                        shortfall \u003e pile._pending and"}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_097830f6","line":2392,"range":{"start_line":2390,"start_character":20,"end_line":2392,"end_character":37},"updated":"2019-07-25 01:29:39.000000000","message":"If we wanted to return non-durable, we\u0027d take this out...","commit_id":"79617532d0b3ea4d31350bd004276c466b446a62"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2404,"context_line":"                               buckets.get_extra_headers)"},{"line_number":2405,"context_line":""},{"line_number":2406,"context_line":"        req.range \u003d orig_range"},{"line_number":2407,"context_line":"        if best_bucket and best_bucket.shortfall \u003c\u003d 0 and best_bucket.durable:"},{"line_number":2408,"context_line":"            # headers can come from any of the getters"},{"line_number":2409,"context_line":"            resp_headers \u003d best_bucket.headers"},{"line_number":2410,"context_line":"            resp_headers.pop(\u0027Content-Range\u0027, None)"}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_297dece6","line":2407,"range":{"start_line":2407,"start_character":54,"end_line":2407,"end_character":77},"updated":"2019-07-25 01:29:39.000000000","message":"... and this. Maybe we could just kinda *stop tracking durability*? Just get me enough conns to reconstruct, dammit!\n\nOh, but maybe we\u0027d still want it to separate not-enough-frags-but-should-be-durable (503) from not-enough-frags-and-not-durable (404).","commit_id":"79617532d0b3ea4d31350bd004276c466b446a62"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b74ed7a5c51c8168d4a15f4fb1c09915e6588175","unresolved":false,"context_lines":[{"line_number":2446,"context_line":"            headers \u003d []"},{"line_number":2447,"context_line":"            for getter, _parts_iter in bad_bucket.get_responses():"},{"line_number":2448,"context_line":"                if best_bucket and best_bucket.durable and \\"},{"line_number":2449,"context_line":"                        not self.app.consider_404s_from_object_handoffs:"},{"line_number":2450,"context_line":"                    headers \u003d HeaderKeyDict(getter.last_headers)"},{"line_number":2451,"context_line":"                    t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":2452,"context_line":"                    t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"7faddb67_9f42cf67","line":2449,"updated":"2019-07-30 13:02:39.000000000","message":"so by default we do not consider 404s from handoffs, so we do handle this block of code.","commit_id":"b29d43397e3ed8ada1c7ffa16120faf193675db6"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"b6b530f712799191a3273f80382654b484003312","unresolved":false,"context_lines":[{"line_number":2794,"context_line":""},{"line_number":2795,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2796,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"},{"line_number":2797,"context_line":"        # which means there is no backend 404 response, but proxy should still"},{"line_number":2798,"context_line":"        # return 404 rather than 503"},{"line_number":2799,"context_line":"        obj1 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj1\u0027)"},{"line_number":2800,"context_line":"        obj2 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj2\u0027)"},{"line_number":2801,"context_line":"        obj3 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj3\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"7faddb67_1e72cc12","line":2798,"range":{"start_line":2797,"start_character":56,"end_line":2798,"end_character":36},"updated":"2019-07-23 11:42:52.000000000","message":"i think this is no longer true, correct?","commit_id":"67f1a6314c6ccb3be61899d1962d7943bad49f93"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be69d4d135bee764863c8575678b5439488902d1","unresolved":false,"context_lines":[{"line_number":2147,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2148,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2149,"context_line":""},{"line_number":2150,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2151,"context_line":""},{"line_number":2152,"context_line":"        # expect a request to all nodes"},{"line_number":2153,"context_line":"        self.assertEqual(2 * self.replicas(), len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_53a83276","line":2150,"updated":"2019-07-23 17:36:39.000000000","message":"it\u0027s pretty obvious to me this test and others were setting up a \"failure to return partial data\" - where the disks had lots of indication that was *something* out there - but couldn\u0027t piece it together.\n\nAnd it\u0027s not like they\u0027re mixing in a tombstone here and there that indicates the problem was probably a \"cleanup issue\" - the test looks more like \"imagine something is wrong in the cluster\"\n\nI think it\u0027s a positive change to represent this to the client as a server error :+1:","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2147,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2148,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2149,"context_line":""},{"line_number":2150,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2151,"context_line":""},{"line_number":2152,"context_line":"        # expect a request to all nodes"},{"line_number":2153,"context_line":"        self.assertEqual(2 * self.replicas(), len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_a9d6bcc2","line":2150,"in_reply_to":"7faddb67_53a83276","updated":"2019-07-25 01:29:39.000000000","message":"Mixing in a tombstone is a good idea... hmm. :-/","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":2794,"context_line":""},{"line_number":2795,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2796,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"},{"line_number":2797,"context_line":"        # which means there is no backend 404 response, but proxy should still"},{"line_number":2798,"context_line":"        # return 404 rather than 503"},{"line_number":2799,"context_line":"        obj1 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj1\u0027)"},{"line_number":2800,"context_line":"        obj2 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj2\u0027)"},{"line_number":2801,"context_line":"        obj3 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj3\u0027)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_13f4fa5a","line":2798,"range":{"start_line":2797,"start_character":56,"end_line":2798,"end_character":36},"updated":"2019-07-23 17:59:25.000000000","message":"Carrying forward Thiago\u0027s comment:\n\n\u003e i think this is no longer true, correct?","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":2838,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2839,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2840,"context_line":""},{"line_number":2841,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2842,"context_line":""},{"line_number":2843,"context_line":"        collected_etags \u003d set()"},{"line_number":2844,"context_line":"        collected_status \u003d set()"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_b39de60b","line":2841,"range":{"start_line":2841,"start_character":42,"end_line":2841,"end_character":45},"updated":"2019-07-23 17:59:25.000000000","message":"Oh, hey! It *already* wasn\u0027t true!","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":2964,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2965,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2966,"context_line":""},{"line_number":2967,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2968,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":2969,"context_line":"        # report having a durable timestamp"},{"line_number":2970,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_f3611e13","line":2967,"range":{"start_line":2967,"start_character":42,"end_line":2967,"end_character":45},"updated":"2019-07-23 17:59:25.000000000","message":"I\u0027m not convinced that this should have changed.","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2964,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2965,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2966,"context_line":""},{"line_number":2967,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2968,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":2969,"context_line":"        # report having a durable timestamp"},{"line_number":2970,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_a9231c05","line":2967,"range":{"start_line":2967,"start_character":42,"end_line":2967,"end_character":45},"in_reply_to":"7faddb67_f3611e13","updated":"2019-07-25 01:29:39.000000000","message":"Done","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be1bbf1e3b482153bfa563dcd8ed61e1109cc991","unresolved":false,"context_lines":[{"line_number":2964,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2965,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2966,"context_line":""},{"line_number":2967,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2968,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":2969,"context_line":"        # report having a durable timestamp"},{"line_number":2970,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_be7ccd72","line":2967,"range":{"start_line":2967,"start_character":42,"end_line":2967,"end_character":45},"in_reply_to":"7faddb67_f3611e13","updated":"2019-07-23 18:05:54.000000000","message":"yeah I\u0027d say it should 200!  I was never quite sure I believed a cluster could have enough non-durable frags to reconstruct and yet not repair in a reclaim age\n\nIn fact, I had argued if we thought that likely we might ought go ahead and have the re-constructor DIG and mark durable if it\u0027s rebuildable!","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":3241,"context_line":"        log_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":3242,"context_line":"        self.assertEqual(log_lines,"},{"line_number":3243,"context_line":"                         [\u0027Problem with fragment response: ETag mismatch\u0027] * 7"},{"line_number":3244,"context_line":"                         + [\u0027Object returning 503 for []\u0027])"},{"line_number":3245,"context_line":"        # Note the empty list above -- that log line comes out of"},{"line_number":3246,"context_line":"        # best_response but we\u0027ve already thrown out the \"good\" responses :-/"},{"line_number":3247,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_13cb9a1e","line":3244,"updated":"2019-07-23 17:59:25.000000000","message":"I\u0027m still not really happy about this log line. We\u0027ve thrown out a *ton* of information that would presumably be useful to an operator.","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be1bbf1e3b482153bfa563dcd8ed61e1109cc991","unresolved":false,"context_lines":[{"line_number":3241,"context_line":"        log_lines \u003d self.app.logger.get_lines_for_level(\u0027error\u0027)"},{"line_number":3242,"context_line":"        self.assertEqual(log_lines,"},{"line_number":3243,"context_line":"                         [\u0027Problem with fragment response: ETag mismatch\u0027] * 7"},{"line_number":3244,"context_line":"                         + [\u0027Object returning 503 for []\u0027])"},{"line_number":3245,"context_line":"        # Note the empty list above -- that log line comes out of"},{"line_number":3246,"context_line":"        # best_response but we\u0027ve already thrown out the \"good\" responses :-/"},{"line_number":3247,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_9e9311a4","line":3244,"in_reply_to":"7faddb67_13cb9a1e","updated":"2019-07-23 18:05:54.000000000","message":"we could log more when dropping the responses...","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be1bbf1e3b482153bfa563dcd8ed61e1109cc991","unresolved":false,"context_lines":[{"line_number":4144,"context_line":"        self.assertEqual({200}, collected_status)"},{"line_number":4145,"context_line":""},{"line_number":4146,"context_line":"    def test_GET_with_no_durable_files(self):"},{"line_number":4147,"context_line":"        # verify that at least one durable is necessary for a successful GET"},{"line_number":4148,"context_line":"        obj1 \u003d self._make_ec_object_stub()"},{"line_number":4149,"context_line":"        node_frags \u003d ["},{"line_number":4150,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 0, \u0027durable\u0027: False},"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_5ee27931","line":4147,"updated":"2019-07-23 18:05:54.000000000","message":"yeah, i\u0027m not sure this was ever correct - but I remember it being a decision we made at time time - same as above worth additional conversation","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","unresolved":false,"context_lines":[{"line_number":4173,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":4174,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":4175,"context_line":""},{"line_number":4176,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":4177,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":4178,"context_line":"        # report having a durable timestamp"},{"line_number":4179,"context_line":"        self.assertEqual(self.replicas() * 2, len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_9e0eb130","line":4176,"range":{"start_line":4176,"start_character":42,"end_line":4176,"end_character":45},"updated":"2019-07-23 17:59:25.000000000","message":"As above, not so sure that this should change.","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":4173,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":4174,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":4175,"context_line":""},{"line_number":4176,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":4177,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":4178,"context_line":"        # report having a durable timestamp"},{"line_number":4179,"context_line":"        self.assertEqual(self.replicas() * 2, len(log))"}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_492068f8","line":4176,"range":{"start_line":4176,"start_character":42,"end_line":4176,"end_character":45},"in_reply_to":"7faddb67_9e0eb130","updated":"2019-07-25 01:29:39.000000000","message":"Done","commit_id":"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dc80187608b3988f2a21e0b92da555ecfebd506c","unresolved":false,"context_lines":[{"line_number":2794,"context_line":""},{"line_number":2795,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2796,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"},{"line_number":2797,"context_line":"        # which means there is no backend 404 response, but proxy should still"},{"line_number":2798,"context_line":"        # return 404 rather than 503"},{"line_number":2799,"context_line":"        obj1 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj1\u0027)"},{"line_number":2800,"context_line":"        obj2 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj2\u0027)"},{"line_number":2801,"context_line":"        obj3 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj3\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_29cbac1e","line":2798,"range":{"start_line":2797,"start_character":56,"end_line":2798,"end_character":36},"updated":"2019-07-25 01:29:39.000000000","message":"Bah -- I still haven\u0027t fixed this outdated (even on master) comment.","commit_id":"79617532d0b3ea4d31350bd004276c466b446a62"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b9f26a36ff9919b4771368d634531d212afd40ba","unresolved":false,"context_lines":[{"line_number":2794,"context_line":""},{"line_number":2795,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2796,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"},{"line_number":2797,"context_line":"        # which means there is no backend 404 response, but proxy should still"},{"line_number":2798,"context_line":"        # return 404 rather than 503"},{"line_number":2799,"context_line":"        obj1 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj1\u0027)"},{"line_number":2800,"context_line":"        obj2 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj2\u0027)"},{"line_number":2801,"context_line":"        obj3 \u003d self._make_ec_object_stub(pattern\u003d\u0027obj3\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_98b8e2e5","line":2798,"range":{"start_line":2797,"start_character":56,"end_line":2798,"end_character":36},"in_reply_to":"7faddb67_29cbac1e","updated":"2019-07-25 15:39:01.000000000","message":"Done","commit_id":"79617532d0b3ea4d31350bd004276c466b446a62"}]}