)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"(In both examples, MD5 gets the raw 16-byte digest while MD5_hex"},{"line_number":18,"context_line":"gets the 32-byte hex-encoded digest.)"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Some clients (such as aws-sdk-java) use the presence of a dash"},{"line_number":21,"context_line":"to decide whether to perform client-side validation of downloads."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"3f79a3b5_b26a3017","line":19,"updated":"2018-08-14 08:13:47.000000000","message":"Around here, I\u0027d like to clarify what\u0027s required for each client actually.\nAnd we would have some options (including pros/cons).","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":17,"context_line":"(In both examples, MD5 gets the raw 16-byte digest while MD5_hex"},{"line_number":18,"context_line":"gets the 32-byte hex-encoded digest.)"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Some clients (such as aws-sdk-java) use the presence of a dash"},{"line_number":21,"context_line":"to decide whether to perform client-side validation of downloads."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Other clients (like s3cmd) use the presence of a dash *in bucket"},{"line_number":24,"context_line":"listings* to decide whether or not to MD5 files locally."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"3f79a3b5_12652446","line":21,"range":{"start_line":20,"start_character":0,"end_line":21,"end_character":65},"updated":"2018-08-14 08:13:47.000000000","message":"Thinking of fixing GETorHEAD response headaer ETag, potentially  we could recalc the ETag from manifest IIF it\u0027s slo and no -\u003c#\u003e suffix.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":20,"context_line":"Some clients (such as aws-sdk-java) use the presence of a dash"},{"line_number":21,"context_line":"to decide whether to perform client-side validation of downloads."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Other clients (like s3cmd) use the presence of a dash *in bucket"},{"line_number":24,"context_line":"listings* to decide whether or not to MD5 files locally."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Now we include a dash as well, to prevent spurious errors like"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"3f79a3b5_92ddb4e1","line":24,"range":{"start_line":23,"start_character":0,"end_line":24,"end_character":56},"updated":"2018-08-14 08:13:47.000000000","message":"I didn\u0027t get how the dash presence is effective for listing.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Now we include a dash as well, to prevent spurious errors like"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"\u003e Unable to verify integrity of data download.  Client calculated"},{"line_number":29,"context_line":"\u003e content hash didn\u0027t match hash calculated by Amazon S3.  The data"},{"line_number":30,"context_line":"\u003e may be corrupt."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"For new multipart-uploads via the S3 API, the ETag that is stored will"},{"line_number":33,"context_line":"be calculated in the same way that AWS uses. This ETag will be used in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"3f79a3b5_f2a0a844","line":30,"range":{"start_line":28,"start_character":0,"end_line":30,"end_character":17},"updated":"2018-08-14 08:13:47.000000000","message":"I think this happens only when the client attempts to download the content (and verify the content via ETag) so it\u0027s not all of this patch attempts to fix.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"}],"swift/common/middleware/s3api/controllers/bucket.py":[{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"3fe884760ffcff66524e41ba8727cfe7584264c7","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                if \u0027s3_etag\u0027 in o:"},{"line_number":190,"context_line":"                    # MUs may already be in the right format"},{"line_number":191,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":192,"context_line":"                elif o[\u0027hash\u0027].startswith(\u0027\"\u0027) and o[\u0027hash\u0027].startswith(\u0027\"\u0027):"},{"line_number":193,"context_line":"                    # Following https://review.openstack.org/#/c/337960/,"},{"line_number":194,"context_line":"                    # SLOs may be in something *close* to the right format"},{"line_number":195,"context_line":"                    etag \u003d o[\u0027hash\u0027][:-1] + \u0027-N\"\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_274627c2","line":192,"range":{"start_line":192,"start_character":61,"end_line":192,"end_character":71},"updated":"2018-06-20 19:08:48.000000000","message":"I\u0027m guessing you mean mean \"endswith\" here.","commit_id":"0bd572888ae98996ed7ad78833aba48b275132ae"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0263f9d637b089b32cc176f8cd7a0f798eea415a","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                if \u0027s3_etag\u0027 in o:"},{"line_number":190,"context_line":"                    # MUs may already be in the right format"},{"line_number":191,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":192,"context_line":"                elif o[\u0027hash\u0027].startswith(\u0027\"\u0027) and o[\u0027hash\u0027].startswith(\u0027\"\u0027):"},{"line_number":193,"context_line":"                    # Following https://review.openstack.org/#/c/337960/,"},{"line_number":194,"context_line":"                    # SLOs may be in something *close* to the right format"},{"line_number":195,"context_line":"                    etag \u003d o[\u0027hash\u0027][:-1] + \u0027-N\"\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_474803b2","line":192,"range":{"start_line":192,"start_character":61,"end_line":192,"end_character":71},"in_reply_to":"5f7c97a3_274627c2","updated":"2018-06-20 19:19:49.000000000","message":"Doh!","commit_id":"0bd572888ae98996ed7ad78833aba48b275132ae"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":185,"context_line":"                SubElement(contents, \u0027Key\u0027).text \u003d o[\u0027name\u0027]"},{"line_number":186,"context_line":"                SubElement(contents, \u0027LastModified\u0027).text \u003d \\"},{"line_number":187,"context_line":"                    o[\u0027last_modified\u0027][:-3] + \u0027Z\u0027"},{"line_number":188,"context_line":"                if \u0027s3_etag\u0027 in o:"},{"line_number":189,"context_line":"                    # MUs may already be in the right format"},{"line_number":190,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":191,"context_line":"                elif o.get(\u0027slo_etag\u0027, \u0027\u0027).endswith(\u0027\"\u0027):"},{"line_number":192,"context_line":"                    # SLOs may be in something *close* to the MU format"},{"line_number":193,"context_line":"                    etag \u003d o[\u0027slo_etag\u0027][:-1] + \u0027-N\"\u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_cf8149c5","line":190,"range":{"start_line":188,"start_character":0,"end_line":190,"end_character":39},"updated":"2018-08-14 08:13:47.000000000","message":"If I understand ListingEtagMiddleware in swift.common.middleware.s3api.s3api, we don\u0027t have to translate the ETag because it\u0027s already translated from s3_etag, isn\u0027t it?","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":185,"context_line":"                SubElement(contents, \u0027Key\u0027).text \u003d o[\u0027name\u0027]"},{"line_number":186,"context_line":"                SubElement(contents, \u0027LastModified\u0027).text \u003d \\"},{"line_number":187,"context_line":"                    o[\u0027last_modified\u0027][:-3] + \u0027Z\u0027"},{"line_number":188,"context_line":"                if \u0027s3_etag\u0027 in o:"},{"line_number":189,"context_line":"                    # MUs may already be in the right format"},{"line_number":190,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":191,"context_line":"                elif o.get(\u0027slo_etag\u0027, \u0027\u0027).endswith(\u0027\"\u0027):"},{"line_number":192,"context_line":"                    # SLOs may be in something *close* to the MU format"},{"line_number":193,"context_line":"                    etag \u003d o[\u0027slo_etag\u0027][:-1] + \u0027-N\"\u0027"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_9d44dc48","line":190,"range":{"start_line":188,"start_character":0,"end_line":190,"end_character":39},"in_reply_to":"3f79a3b5_cf8149c5","updated":"2018-08-14 17:25:19.000000000","message":"Correct. We just have to make sure we grab the right key out of the object dict.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":188,"context_line":"                if \u0027s3_etag\u0027 in o:"},{"line_number":189,"context_line":"                    # MUs may already be in the right format"},{"line_number":190,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":191,"context_line":"                elif o.get(\u0027slo_etag\u0027, \u0027\u0027).endswith(\u0027\"\u0027):"},{"line_number":192,"context_line":"                    # SLOs may be in something *close* to the MU format"},{"line_number":193,"context_line":"                    etag \u003d o[\u0027slo_etag\u0027][:-1] + \u0027-N\"\u0027"},{"line_number":194,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_2216c756","line":191,"range":{"start_line":191,"start_character":42,"end_line":191,"end_character":56},"updated":"2018-08-14 17:25:19.000000000","message":"Not really needed -- we can just check for the presence of a \u0027slo_etag\u0027 key, .strip(\u0027\"\u0027), and reinstate the quotes.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                    # MUs may already be in the right format"},{"line_number":190,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":191,"context_line":"                elif o.get(\u0027slo_etag\u0027, \u0027\u0027).endswith(\u0027\"\u0027):"},{"line_number":192,"context_line":"                    # SLOs may be in something *close* to the MU format"},{"line_number":193,"context_line":"                    etag \u003d o[\u0027slo_etag\u0027][:-1] + \u0027-N\"\u0027"},{"line_number":194,"context_line":"                else:"},{"line_number":195,"context_line":"                    etag \u003d \u0027\"%s\"\u0027 % o[\u0027hash\u0027]"},{"line_number":196,"context_line":"                SubElement(contents, \u0027ETag\u0027).text \u003d etag"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_724ed838","line":193,"range":{"start_line":192,"start_character":0,"end_line":193,"end_character":53},"updated":"2018-08-14 08:13:47.000000000","message":"To get this patch merged asap, I\u0027d like to separate the patch on how we deal with the slo via native swift api. This doesn\u0027t seem trivial to me.\n\nIf we would like to get the best compatibility, perhaps, we could add one more backend request retrieving the manifest to calculate \"correct\" s3 etag. I know it makes more performance overhead and complexity with how to check the etag on if-match/if-non-match request tho. I know.. I know... on the other hand, \"-N\" looks also brittle fix that S3 should not append it natively.\n\nOne possible idea is like changing GETorHEAD steps likely,\n\n1. Call GETorHEAD without if-match changes\n2. if the request got 412 precondition failed then, it\u0027s X-Static-Large-Object\u003dTrue, assert the s3 etag with fetching manifest and recalc at s3 layer\n3. then, if it matches the etag client sent, continue to GETorHEAD with if-match for slo that is fetched at step 2.\n\nThat makes more extra backend requests but it will happen if and only if user creates slo via Swift API and user calls GETorHEAD with if-match against to that object.\n\nI may be missing any other corner case tho...\n\nThought?","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                    # MUs may already be in the right format"},{"line_number":190,"context_line":"                    etag \u003d o[\u0027s3_etag\u0027]"},{"line_number":191,"context_line":"                elif o.get(\u0027slo_etag\u0027, \u0027\u0027).endswith(\u0027\"\u0027):"},{"line_number":192,"context_line":"                    # SLOs may be in something *close* to the MU format"},{"line_number":193,"context_line":"                    etag \u003d o[\u0027slo_etag\u0027][:-1] + \u0027-N\"\u0027"},{"line_number":194,"context_line":"                else:"},{"line_number":195,"context_line":"                    etag \u003d \u0027\"%s\"\u0027 % o[\u0027hash\u0027]"},{"line_number":196,"context_line":"                SubElement(contents, \u0027ETag\u0027).text \u003d etag"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_5d53a417","line":193,"range":{"start_line":192,"start_character":0,"end_line":193,"end_character":53},"in_reply_to":"3f79a3b5_724ed838","updated":"2018-08-14 17:25:19.000000000","message":"The biggest reservation I have about adding backend requests is that we\u0027re in the *bucket* controller right here -- adding a new GET for every multipart upload could mean a thousand requests...\n\nThinking *just* about GETorHEAD, this still makes me nervous:\n\n\u003e if the request got 412 precondition failed then, it\u0027s X-Static-Large-Object\u003dTrue\n\nI\u0027d need to check whether we include X-Static-Large-Object in a 412... IIRC we at least include sysmeta, but I\u0027m not sure about that header...","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"}],"swift/common/middleware/s3api/controllers/obj.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":69,"context_line":"            for value in list_from_csv(req.headers[match_header]):"},{"line_number":70,"context_line":"                if value.startswith(\u0027\"\u0027) and value.endswith(\u0027\"\u0027):"},{"line_number":71,"context_line":"                    value \u003d value[1:-1]"},{"line_number":72,"context_line":"                if value.endswith(\u0027-N\u0027):"},{"line_number":73,"context_line":"                    # Deal with fake S3-like etags for SLOs uploaded via Swift"},{"line_number":74,"context_line":"                    req.headers[match_header] +\u003d \u0027, \u0027 + value[:-2]"},{"line_number":75,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_af672de9","line":72,"updated":"2018-08-14 08:13:47.000000000","message":"What\u0027s?\n\nI\u0027m not sure if some s3 clients requests if-match \"-N\" ?\nIt might happen that a client send the etag from listing result... tho.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":69,"context_line":"            for value in list_from_csv(req.headers[match_header]):"},{"line_number":70,"context_line":"                if value.startswith(\u0027\"\u0027) and value.endswith(\u0027\"\u0027):"},{"line_number":71,"context_line":"                    value \u003d value[1:-1]"},{"line_number":72,"context_line":"                if value.endswith(\u0027-N\u0027):"},{"line_number":73,"context_line":"                    # Deal with fake S3-like etags for SLOs uploaded via Swift"},{"line_number":74,"context_line":"                    req.headers[match_header] +\u003d \u0027, \u0027 + value[:-2]"},{"line_number":75,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_a27ad7b6","line":72,"in_reply_to":"3f79a3b5_af672de9","updated":"2018-08-14 17:25:19.000000000","message":"See swift/common/middleware/s3api/s3response.py -- if we see a normal SLO (not an SLO that got created to complete a MU), we\u0027ll tack on a -N. This lets us \"fix\" pre-existing data in the cluster.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"}],"swift/common/middleware/s3api/s3api.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"74e33187e8ec9f7f343b655b2ae5f412a5411155","unresolved":false,"context_lines":[{"line_number":95,"context_line":"from swift.common.middleware.s3api.s3request import get_request_class"},{"line_number":96,"context_line":"from swift.common.middleware.s3api.s3response import ErrorResponse, \\"},{"line_number":97,"context_line":"    InternalError, MethodNotAllowed, S3ResponseBase, S3NotImplemented"},{"line_number":98,"context_line":"from swift.common.utils import get_logger, register_swift_info, \\"},{"line_number":99,"context_line":"    config_true_value, config_positive_int_value, split_path"},{"line_number":100,"context_line":"from swift.common.middleware.s3api.utils import Config"},{"line_number":101,"context_line":"from swift.common.middleware.s3api.acl_handlers import get_acl_handler"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_43ded411","line":98,"updated":"2018-08-09 06:26:02.000000000","message":"We aready added \"from swift.common.utils\" above, at line 90... Not a big deal, but looks odd.","commit_id":"52219a82a700faaf0f0e9088c0b7dcc0b9af9656"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d85cf4e0d92aa47943c8d776a54f5142cf92a497","unresolved":false,"context_lines":[{"line_number":95,"context_line":"from swift.common.middleware.s3api.s3request import get_request_class"},{"line_number":96,"context_line":"from swift.common.middleware.s3api.s3response import ErrorResponse, \\"},{"line_number":97,"context_line":"    InternalError, MethodNotAllowed, S3ResponseBase, S3NotImplemented"},{"line_number":98,"context_line":"from swift.common.utils import get_logger, register_swift_info, \\"},{"line_number":99,"context_line":"    config_true_value, config_positive_int_value, split_path"},{"line_number":100,"context_line":"from swift.common.middleware.s3api.utils import Config"},{"line_number":101,"context_line":"from swift.common.middleware.s3api.acl_handlers import get_acl_handler"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_5afdfc6b","line":98,"in_reply_to":"3f79a3b5_43ded411","updated":"2018-08-13 21:22:12.000000000","message":"Done","commit_id":"52219a82a700faaf0f0e9088c0b7dcc0b9af9656"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":140,"context_line":"                    break"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"        if content_type !\u003d \u0027application/json\u0027 or content_length is None or \\"},{"line_number":143,"context_line":"                content_length \u003e 2 * 1024 * 1000:"},{"line_number":144,"context_line":"            start_response(ctx._response_status, ctx._response_headers,"},{"line_number":145,"context_line":"                           ctx._response_exc_info)"},{"line_number":146,"context_line":"            return resp_iter"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_0fbbc1c4","line":143,"range":{"start_line":143,"start_character":33,"end_line":143,"end_character":48},"updated":"2018-08-14 08:13:47.000000000","message":"Why is this different from MAX_CONTAINER_LISTING_CONTENT_LENGTH at listing_formats middleware?","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":140,"context_line":"                    break"},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"        if content_type !\u003d \u0027application/json\u0027 or content_length is None or \\"},{"line_number":143,"context_line":"                content_length \u003e 2 * 1024 * 1000:"},{"line_number":144,"context_line":"            start_response(ctx._response_status, ctx._response_headers,"},{"line_number":145,"context_line":"                           ctx._response_exc_info)"},{"line_number":146,"context_line":"            return resp_iter"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_e201af39","line":143,"range":{"start_line":143,"start_character":33,"end_line":143,"end_character":48},"in_reply_to":"3f79a3b5_0fbbc1c4","updated":"2018-08-14 17:25:19.000000000","message":"I\u0027m guessing I was thinking that since S3 responses will be capped at 1,000 items instead of 10,000, we could use a lower limit... but you\u0027re right, I should piggy-back off of that, particularly since this is supposed to apply to *all* Swift responses.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":345,"context_line":"    )"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def s3api_filter(app):"},{"line_number":348,"context_line":"        return S3ApiMiddleware(ListingEtagMiddleware(app), conf)"},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"    return s3api_filter"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_8f2a71d8","line":348,"range":{"start_line":348,"start_character":31,"end_line":348,"end_character":57},"updated":"2018-08-14 08:13:47.000000000","message":"Honesty, if it could be, I\u0027d like to re-use (or expand) listing_formats middleware for this use case instead of rewrite almost same code with that, here.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":345,"context_line":"    )"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def s3api_filter(app):"},{"line_number":348,"context_line":"        return S3ApiMiddleware(ListingEtagMiddleware(app), conf)"},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"    return s3api_filter"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_0282abc8","line":348,"range":{"start_line":348,"start_character":31,"end_line":348,"end_character":57},"in_reply_to":"3f79a3b5_8f2a71d8","updated":"2018-08-14 17:25:19.000000000","message":"I\u0027m not sure that I like having listing_formats know about s3api implementation details, though. Similar to how SLO is responsible for breaking out slo_etag [1] or how the decrypter is responsible for decrypting the hash [2]. It seems to me that whoever did the original manipulation ought to be responsible for getting us back to a sane response.\n\n[1] https://github.com/openstack/swift/blob/d1675d0/swift/common/middleware/slo.py#L1438-L1441\n[2] https://github.com/openstack/swift/blob/2.18.0/swift/common/middleware/crypto/decrypter.py#L379-L396","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1e391ccffe34a35efa310033f8e4e4df70053f2b","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        )"},{"line_number":175,"context_line":"        start_response(ctx._response_status, ctx._response_headers,"},{"line_number":176,"context_line":"                       ctx._response_exc_info)"},{"line_number":177,"context_line":"        return [body]"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"class S3ApiMiddleware(object):"}],"source_content_type":"text/x-python","patch_set":11,"id":"3f79a3b5_4a012e04","line":177,"updated":"2018-09-12 16:12:35.000000000","message":"Oh, what? Is this the thing for cross compatibility APIs?\n\nWIthout the ListingMiddleware, just a few changes as follows gets worked.\n\n--- a/swift/common/middleware/s3api/controllers/bucket.py\n+++ b/swift/common/middleware/s3api/controllers/bucket.py\n@@ -13,6 +13,7 @@\n # See the License for the specific language governing permissions and\n # limitations under the License.\n \n+from cgi import parse_header\n from base64 import standard_b64encode as b64encode\n from base64 import standard_b64decode as b64decode\n \n@@ -185,11 +186,13 @@ class BucketController(Controller):\n                 SubElement(contents, \u0027Key\u0027).text \u003d o[\u0027name\u0027]\n                 SubElement(contents, \u0027LastModified\u0027).text \u003d \\\n                     o[\u0027last_modified\u0027][:-3] + \u0027Z\u0027\n-                if \u0027s3_etag\u0027 in o:\n-                    # New-enough MUs are already in the right format\n-                    etag \u003d o[\u0027s3_etag\u0027]\n+\n+                value, params \u003d parse_header(o[\u0027hash\u0027])\n+                if \u0027s3_etag\u0027 in params:\n+                    etag \u003d \u0027\"%s\"\u0027 % params.pop(\u0027s3_etag\u0027)\n                 else:\n-                    etag \u003d \u0027\"%s\"\u0027 % o[\u0027hash\u0027]\n+                    etag \u003d value + \u0027\u0027.join(\n+                        \u0027; %s\u003d%s\u0027 % kv for kv in params.items())\n                 SubElement(contents, \u0027ETag\u0027).text \u003d etag\n                 SubElement(contents, \u0027Size\u0027).text \u003d str(o[\u0027bytes\u0027])\n                 if fetch_owner or not is_v2:\ndiff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py","commit_id":"321694b130aafef165e0b7dab1ff0b66276f75f6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2e639b2b4a7f22876d0d20052c4900f1c4309b05","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        )"},{"line_number":175,"context_line":"        start_response(ctx._response_status, ctx._response_headers,"},{"line_number":176,"context_line":"                       ctx._response_exc_info)"},{"line_number":177,"context_line":"        return [body]"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"class S3ApiMiddleware(object):"}],"source_content_type":"text/x-python","patch_set":11,"id":"3f79a3b5_b2b625f9","line":177,"in_reply_to":"3f79a3b5_4a012e04","updated":"2018-09-13 07:54:10.000000000","message":"Yeah, we want to get the s3_etag out of the hash for normal swift requests. Then, as long as we\u0027ve got it anyway, may as well add it to all responses, even swift-only ones. I\u0027ll write up a unit test for it.","commit_id":"321694b130aafef165e0b7dab1ff0b66276f75f6"}],"swift/common/middleware/s3api/s3response.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":143,"context_line":"            # to attempt client-side download validation, so even if we"},{"line_number":144,"context_line":"            # didn\u0027t store the AWS-style header, tack on a \u0027-N\u0027. (Use \u0027N\u0027"},{"line_number":145,"context_line":"            # because we don\u0027t actually know how many parts there are.)"},{"line_number":146,"context_line":"            headers[\u0027etag\u0027] +\u003d \u0027-N\u0027"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"        self.headers \u003d headers"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_e73c6130","line":146,"updated":"2018-08-14 17:25:19.000000000","message":"This is where S3 clients might get the idea that an ETag should have \u0027-N\u0027 at the end.","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e2dc35bfd18c385af8c77c232d0fd3a52ea12677","unresolved":false,"context_lines":[{"line_number":143,"context_line":"            # to attempt client-side download validation, so even if we"},{"line_number":144,"context_line":"            # didn\u0027t store the AWS-style header, tack on a \u0027-N\u0027. (Use \u0027N\u0027"},{"line_number":145,"context_line":"            # because we don\u0027t actually know how many parts there are.)"},{"line_number":146,"context_line":"            headers[\u0027etag\u0027] +\u003d \u0027-N\u0027"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"        self.headers \u003d headers"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_6b3c77e0","line":146,"in_reply_to":"3f79a3b5_e73c6130","updated":"2018-08-16 04:57:10.000000000","message":"While I was in dayoff, I\u0027m realizing, perhaps, we could pass the slo information from slo middleware via the request environ object? Something like\n\n # assuming slo middlware sets the manifest info somewhere like environ[\"swift.slo_manifest\"]\n slo_manifest \u003d req.environ[\u0027swift.slo_manifest\u0027]\n num_parts \u003d len(slo_mfanifest)\n\nAh... it might needs more consideration on how it works with nested slo tho...","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"}],"test/functional/s3api/test_multi_upload.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e18108abb9139ce8136db0b22340dee8659725a5","unresolved":false,"context_lines":[{"line_number":312,"context_line":"            elem.find(\u0027Location\u0027).text)"},{"line_number":313,"context_line":"        self.assertEqual(elem.find(\u0027Bucket\u0027).text, bucket)"},{"line_number":314,"context_line":"        self.assertEqual(elem.find(\u0027Key\u0027).text, key)"},{"line_number":315,"context_line":"        concatted_etags \u003d \u0027\u0027.join(etag.strip(\u0027\"\u0027) for etag in etags)"},{"line_number":316,"context_line":"        exp_etag \u003d \u0027\"%s-%s\"\u0027 % ("},{"line_number":317,"context_line":"            md5(concatted_etags.decode(\u0027hex\u0027)).hexdigest(), len(etags))"},{"line_number":318,"context_line":"        etag \u003d elem.find(\u0027ETag\u0027).text"},{"line_number":319,"context_line":"        self.assertEqual(etag, exp_etag)"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_4cf2279c","line":317,"range":{"start_line":315,"start_character":0,"end_line":317,"end_character":71},"updated":"2018-08-14 08:13:47.000000000","message":"I confirmed locally, that this concatenated etag for each part with decoding hex, then hashed via md5 results in match with Amazon S3 Etag for multipart uploaded object. (At least, us standard follows this formula)","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e2dc35bfd18c385af8c77c232d0fd3a52ea12677","unresolved":false,"context_lines":[{"line_number":312,"context_line":"            elem.find(\u0027Location\u0027).text)"},{"line_number":313,"context_line":"        self.assertEqual(elem.find(\u0027Bucket\u0027).text, bucket)"},{"line_number":314,"context_line":"        self.assertEqual(elem.find(\u0027Key\u0027).text, key)"},{"line_number":315,"context_line":"        concatted_etags \u003d \u0027\u0027.join(etag.strip(\u0027\"\u0027) for etag in etags)"},{"line_number":316,"context_line":"        exp_etag \u003d \u0027\"%s-%s\"\u0027 % ("},{"line_number":317,"context_line":"            md5(concatted_etags.decode(\u0027hex\u0027)).hexdigest(), len(etags))"},{"line_number":318,"context_line":"        etag \u003d elem.find(\u0027ETag\u0027).text"},{"line_number":319,"context_line":"        self.assertEqual(etag, exp_etag)"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_8b1c7378","line":317,"range":{"start_line":315,"start_character":0,"end_line":317,"end_character":71},"in_reply_to":"3f79a3b5_1d720cae","updated":"2018-08-16 04:57:10.000000000","message":":-/","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8c8b5bfba696e23707637271ec381f278891c9e9","unresolved":false,"context_lines":[{"line_number":312,"context_line":"            elem.find(\u0027Location\u0027).text)"},{"line_number":313,"context_line":"        self.assertEqual(elem.find(\u0027Bucket\u0027).text, bucket)"},{"line_number":314,"context_line":"        self.assertEqual(elem.find(\u0027Key\u0027).text, key)"},{"line_number":315,"context_line":"        concatted_etags \u003d \u0027\u0027.join(etag.strip(\u0027\"\u0027) for etag in etags)"},{"line_number":316,"context_line":"        exp_etag \u003d \u0027\"%s-%s\"\u0027 % ("},{"line_number":317,"context_line":"            md5(concatted_etags.decode(\u0027hex\u0027)).hexdigest(), len(etags))"},{"line_number":318,"context_line":"        etag \u003d elem.find(\u0027ETag\u0027).text"},{"line_number":319,"context_line":"        self.assertEqual(etag, exp_etag)"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_1d720cae","line":317,"range":{"start_line":315,"start_character":0,"end_line":317,"end_character":71},"in_reply_to":"3f79a3b5_4cf2279c","updated":"2018-08-14 17:25:19.000000000","message":"\u003e (At least, us standard follows this formula)\n\nI hate that we always have to add caveats like that to our S3-compat statements :-/","commit_id":"19dc08b4f1fa870bb6325043e60d1ed8493dd576"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1e391ccffe34a35efa310033f8e4e4df70053f2b","unresolved":false,"context_lines":[{"line_number":348,"context_line":""},{"line_number":349,"context_line":"        # More interesting conditional cases"},{"line_number":350,"context_line":"        # TODO: this requires https://review.openstack.org/#/c/556683/ to work"},{"line_number":351,"context_line":"        # check_obj({\u0027If-Match\u0027: exp_etag}, 200)"},{"line_number":352,"context_line":"        # In light of above -- SLO responds 200 but we translate to 412"},{"line_number":353,"context_line":"        check_obj({\u0027If-Match\u0027: swift_etag}, 412)"},{"line_number":354,"context_line":"        # In light of above -- I\u0027m not sure why this passes"}],"source_content_type":"text/x-python","patch_set":11,"id":"3f79a3b5_0a11f604","line":351,"updated":"2018-09-12 16:12:35.000000000","message":"The requirement already done so we should plug the comment here.","commit_id":"321694b130aafef165e0b7dab1ff0b66276f75f6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2e639b2b4a7f22876d0d20052c4900f1c4309b05","unresolved":false,"context_lines":[{"line_number":348,"context_line":""},{"line_number":349,"context_line":"        # More interesting conditional cases"},{"line_number":350,"context_line":"        # TODO: this requires https://review.openstack.org/#/c/556683/ to work"},{"line_number":351,"context_line":"        # check_obj({\u0027If-Match\u0027: exp_etag}, 200)"},{"line_number":352,"context_line":"        # In light of above -- SLO responds 200 but we translate to 412"},{"line_number":353,"context_line":"        check_obj({\u0027If-Match\u0027: swift_etag}, 412)"},{"line_number":354,"context_line":"        # In light of above -- I\u0027m not sure why this passes"}],"source_content_type":"text/x-python","patch_set":11,"id":"3f79a3b5_f2057dc2","line":351,"in_reply_to":"3f79a3b5_0a11f604","updated":"2018-09-13 07:54:10.000000000","message":"Done","commit_id":"321694b130aafef165e0b7dab1ff0b66276f75f6"}],"test/unit/common/middleware/s3api/test_bucket.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"1e391ccffe34a35efa310033f8e4e4df70053f2b","unresolved":false,"context_lines":[{"line_number":44,"context_line":"        objects \u003d ["},{"line_number":45,"context_line":"            {\u0027name\u0027: str(item[0]), \u0027last_modified\u0027: str(item[1]),"},{"line_number":46,"context_line":"             \u0027hash\u0027: str(item[2]), \u0027bytes\u0027: str(item[3])}"},{"line_number":47,"context_line":"            for item in self.objects]"},{"line_number":48,"context_line":"        object_list \u003d json.dumps(objects)"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"        self.prefixes \u003d [\u0027rose\u0027, \u0027viola\u0027, \u0027lily\u0027]"}],"source_content_type":"text/x-python","patch_set":11,"id":"3f79a3b5_aacc828a","line":47,"updated":"2018-09-12 16:12:35.000000000","message":"good cleanup","commit_id":"321694b130aafef165e0b7dab1ff0b66276f75f6"}],"test/unit/common/middleware/s3api/test_s3api.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"267047ed917dede1885315dfebd2a548bef598cd","unresolved":false,"context_lines":[{"line_number":47,"context_line":"        # This translation happens all the time, even on normal swift requests"},{"line_number":48,"context_line":"        body_data \u003d  json.dumps(["},{"line_number":49,"context_line":"            {\u0027name\u0027: \u0027obj1\u0027, \u0027hash\u0027: \u00270123456789abcdef0123456789abcdef\u0027},"},{"line_number":50,"context_line":"            {\u0027name\u0027: \u0027obj2\u0027, \u0027hash\u0027: \u0027swiftetag; s3_etag\u003dmu-etag\u0027},"},{"line_number":51,"context_line":"            {\u0027name\u0027: \u0027obj2\u0027, \u0027hash\u0027: \u0027swiftetag; something\u003delse\u0027},"},{"line_number":52,"context_line":"            {\u0027subdir\u0027: \u0027path/\u0027},"},{"line_number":53,"context_line":"        ]).encode(\u0027ascii\u0027)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_a800f867","line":50,"range":{"start_line":50,"start_character":29,"end_line":50,"end_character":67},"updated":"2018-09-13 10:33:00.000000000","message":"OK. That make sense, s3api has the responsibility to parse the mangled hash. Otherwise, Swift API could break something.","commit_id":"18ec621d9be7d5d8ad405d9cd03c658fd81edee1"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"267047ed917dede1885315dfebd2a548bef598cd","unresolved":false,"context_lines":[{"line_number":83,"context_line":"        ]).encode(\u0027ascii\u0027)"},{"line_number":84,"context_line":"        self.swift.register("},{"line_number":85,"context_line":"            \u0027GET\u0027, \u0027/v1/a/c\u0027, swob.HTTPOk,"},{"line_number":86,"context_line":"            {\u0027Content-Type\u0027: \u0027text/plain; charset\u003dUTF-8\u0027},"},{"line_number":87,"context_line":"            body_data)"},{"line_number":88,"context_line":"        req \u003d Request.blank(\u0027/v1/a/c\u0027)"},{"line_number":89,"context_line":"        status, headers, body \u003d self.call_s3api(req)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_88e41ce2","line":86,"range":{"start_line":86,"start_character":28,"end_line":86,"end_character":45},"updated":"2018-09-13 10:33:00.000000000","message":"OK. this is text/plain.","commit_id":"18ec621d9be7d5d8ad405d9cd03c658fd81edee1"}]}