)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"1a0d0f23_89ccb3e7","updated":"2023-11-22 22:45:18.000000000","message":"i\u0027m sure that we should return 503 on 409 from DELETE marker request during complete-mpu-upload\n\nI\u0027m less sure about the most obvious way to do that and if we should ask the client to reconnect.","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"}],"swift/common/middleware/s3api/controllers/multi_upload.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":true,"context_lines":[{"line_number":68,"context_line":"import six"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"from swift.common import constraints"},{"line_number":71,"context_line":"from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, wsgi_to_str"},{"line_number":72,"context_line":"from swift.common.utils import json, public, reiterate, md5"},{"line_number":73,"context_line":"from swift.common.db import utf8encode"},{"line_number":74,"context_line":"from swift.common.request_helpers import get_container_update_override_key, \\"}],"source_content_type":"text/x-python","patch_set":1,"id":"4f32ee7b_6f076ddc","line":71,"updated":"2023-11-22 22:45:18.000000000","message":"heh, I didn\u0027t even notice this at first until I was reverting it - my editor says the line was 79 wide, so borderline","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"}],"swift/common/middleware/s3api/s3request.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":true,"context_lines":[{"line_number":1432,"context_line":"                raise SlowDown(status\u003d\u0027429 Slow Down\u0027)"},{"line_number":1433,"context_line":"            raise SlowDown()"},{"line_number":1434,"context_line":"        if resp.status_int \u003d\u003d HTTP_CONFLICT:"},{"line_number":1435,"context_line":"            if self.method \u003d\u003d \u0027GET\u0027:"},{"line_number":1436,"context_line":"                raise BrokenMPU()"},{"line_number":1437,"context_line":""},{"line_number":1438,"context_line":"        raise InternalError(\u0027unexpected status code %d\u0027 % status)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ccef4941_4b3c1456","line":1435,"updated":"2023-11-22 22:45:18.000000000","message":"GET method (and maybe slo sysmeta in resp headers) would be a good reason to assume it\u0027s a 409 from SLO because of missing parts.\n\nThere\u0027s LOTs of other 409 errors listed in https://github.com/NVIDIA/swift/blob/master/swift/common/middleware/s3api/s3response.py I think BrokenMPU is the only one we make up?","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":true,"context_lines":[{"line_number":967,"context_line":"                    \u0027s3api-test-key-[0-9a-f]*/[a-zA-Z0-9]{48}$\u0027, req.path):"},{"line_number":968,"context_line":"            self.logger.debug(\u0027found marker DELETE %s\u0027 % req.path)"},{"line_number":969,"context_line":"            # XXX maybe this proxy is behind the times"},{"line_number":970,"context_line":"            req.headers[\u0027X-Timestamp\u0027] \u003d Timestamp(time.time() - 18.0).normal"},{"line_number":971,"context_line":"        req.ensure_x_timestamp()"},{"line_number":972,"context_line":""},{"line_number":973,"context_line":"        # Include local handoff nodes if write-affinity is enabled."}],"source_content_type":"text/x-python","patch_set":1,"id":"86caaf02_1fd8e18b","line":970,"updated":"2023-11-22 22:45:18.000000000","message":"this will normally fail with the default 4 retries in aws-cli, I think in prod we hypothizied it might have been as bad as ~5s\n\nyou can try lowering it, in the vsaio test the reconnect doesn\u0027t matter cause you always connect back to this \"bad node\"\n\nbut client backoff sleep larger than this value helps!  You can try going to a smaller/less-bad value here if you want to see the client succeed more often.","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"}],"test/s3api/test_mpu.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":true,"context_lines":[{"line_number":68,"context_line":"                json.dumps({\u0027Parts\u0027: parts}),"},{"line_number":69,"context_line":"            )"},{"line_number":70,"context_line":"        print(\u0027\\n%s\u0027 % cmd)"},{"line_number":71,"context_line":"        # os.system(cmd)"},{"line_number":72,"context_line":"        complete_mpu_resp \u003d self.client.complete_multipart_upload("},{"line_number":73,"context_line":"            Bucket\u003dself.bucket_name, Key\u003dkey_name,"},{"line_number":74,"context_line":"            MultipartUpload\u003d{"}],"source_content_type":"text/x-python","patch_set":1,"id":"77d8f4f6_4a7570ca","line":71,"updated":"2023-11-22 22:45:18.000000000","message":"at first I using a debugger to break here and copy and pasting the command to shell but with low values of clock drift it was hard to get there fast enough - eventually I just decided to have the test run it for me.","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6b3b7cb0810052e5464178efa23ee92c38658dfe","unresolved":true,"context_lines":[{"line_number":75,"context_line":"                \u0027Parts\u0027: parts,"},{"line_number":76,"context_line":"            },"},{"line_number":77,"context_line":"            UploadId\u003dupload_id,"},{"line_number":78,"context_line":"        )"},{"line_number":79,"context_line":"        self.assertEqual(200, complete_mpu_resp["},{"line_number":80,"context_line":"            \u0027ResponseMetadata\u0027][\u0027HTTPStatusCode\u0027])"},{"line_number":81,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"e55210d4_3bcacbaf","line":78,"updated":"2023-11-22 22:45:18.000000000","message":"FWIW there\u0027s some retry/backoff on 503 baked into this client too - but it ALSO doesn\u0027t retry on the 409.","commit_id":"ce3d2f6b3dde6201c5efd895863861a8a7e2f434"}]}