)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":17,"context_line":"   be invalid"},{"line_number":18,"context_line":"3. so params[\u0027xxx\u0027] getter must return values in UTF-8"},{"line_number":19,"context_line":"4. so we modified the params(), have _param_cache pre-decoded;"},{"line_number":20,"context_line":"   the getter returns utf-8 now"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: Ia5815602d05925c5de110accc4dfb1368203bd8d"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3f79a3b5_dba012f5","line":20,"updated":"2018-10-26 23:30:24.000000000","message":"Seems like something that check_utf8(wsgi_to_str(param)) could handle... I know we already do something like that in common.constraints.check_metadata\n\nThat said, I\u0027m much more inclined to allow manipulations in the helper dict we return; much better than actually tweaking things in the environment dict.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":17,"context_line":"   be invalid"},{"line_number":18,"context_line":"3. so params[\u0027xxx\u0027] getter must return values in UTF-8"},{"line_number":19,"context_line":"4. so we modified the params(), have _param_cache pre-decoded;"},{"line_number":20,"context_line":"   the getter returns utf-8 now"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: Ia5815602d05925c5de110accc4dfb1368203bd8d"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3f79a3b5_ffa31daf","line":20,"in_reply_to":"3f79a3b5_826c1f06","updated":"2018-11-01 17:10:19.000000000","message":"Actually, I was thinking of wsgi_to_str. Pretty sure either should work with check_utf8 though; wsgi_to_str would take invalid bytes and turn them into unpaired surrogates which would fail the check.\n\nI can see that I didn\u0027t flesh out my idea enough, though -- my thought was that since environ[\u0027QUERY_STRING\u0027] is a wsgi string (must be; no getting around that), req.params should be populated with wsgi strings. Then request_helpers.get_param could do something like\n\n value \u003d wsgi_to_str(req.params.get(name, default))\n if not check_utf8(value):\n     raise HTTPBadRequest(...)\n return value\n\n(Maybe with a `default \u003d str_to_wsgi(default)` first, but it seems kinda unlikely that any caller would use a non-ascii default.)","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":17,"context_line":"   be invalid"},{"line_number":18,"context_line":"3. so params[\u0027xxx\u0027] getter must return values in UTF-8"},{"line_number":19,"context_line":"4. so we modified the params(), have _param_cache pre-decoded;"},{"line_number":20,"context_line":"   the getter returns utf-8 now"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: Ia5815602d05925c5de110accc4dfb1368203bd8d"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3f79a3b5_826c1f06","line":20,"in_reply_to":"3f79a3b5_dba012f5","updated":"2018-11-01 01:58:14.000000000","message":"Did you mean wsgi_to_bytes above? Even so, the problem is that at times, we need to use broken UTF-8 strings. We actually have a test where byte 0xcc is used as a delimiter. At other times, we reject those. The knowledge if these need to be rejected is not available when params() is invoked. This is the real struggle.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"}],"swift/account/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":398,"context_line":"            if isinstance(prefix, bytes):"},{"line_number":399,"context_line":"                prefix \u003d prefix.decode(\u0027utf-8\u0027)"},{"line_number":400,"context_line":"            if isinstance(delimiter, bytes):"},{"line_number":401,"context_line":"                delimiter \u003d delimiter.decode(\u0027utf-8\u0027)"},{"line_number":402,"context_line":"        if reverse:"},{"line_number":403,"context_line":"            # Reverse the markers if we are reversing the listing."},{"line_number":404,"context_line":"            marker, end_marker \u003d end_marker, marker"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_a86e2efb","line":401,"updated":"2018-10-26 23:30:24.000000000","message":"I was considering rolling this into common.db.utf8encode, but then I guess we\u0027d need to think of a better name...","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":398,"context_line":"            if isinstance(prefix, bytes):"},{"line_number":399,"context_line":"                prefix \u003d prefix.decode(\u0027utf-8\u0027)"},{"line_number":400,"context_line":"            if isinstance(delimiter, bytes):"},{"line_number":401,"context_line":"                delimiter \u003d delimiter.decode(\u0027utf-8\u0027)"},{"line_number":402,"context_line":"        if reverse:"},{"line_number":403,"context_line":"            # Reverse the markers if we are reversing the listing."},{"line_number":404,"context_line":"            marker, end_marker \u003d end_marker, marker"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_e244f378","line":401,"in_reply_to":"3f79a3b5_a86e2efb","updated":"2018-11-01 01:58:14.000000000","message":"Yeah... Maybe later.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":406,"context_line":"        if delimiter and not prefix:"},{"line_number":407,"context_line":"            prefix \u003d \u0027\u0027"},{"line_number":408,"context_line":"        if prefix:"},{"line_number":409,"context_line":"            end_prefix \u003d prefix[:-1] + chr(ordx(prefix[-1]) + 1)"},{"line_number":410,"context_line":"        orig_marker \u003d marker"},{"line_number":411,"context_line":"        with self.get() as conn:"},{"line_number":412,"context_line":"            results \u003d []"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_c83e0ae2","line":409,"updated":"2018-10-26 23:30:24.000000000","message":"Alternatively, chr(ord(prefix[-1:]) + 1)","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":406,"context_line":"        if delimiter and not prefix:"},{"line_number":407,"context_line":"            prefix \u003d \u0027\u0027"},{"line_number":408,"context_line":"        if prefix:"},{"line_number":409,"context_line":"            end_prefix \u003d prefix[:-1] + chr(ordx(prefix[-1]) + 1)"},{"line_number":410,"context_line":"        orig_marker \u003d marker"},{"line_number":411,"context_line":"        with self.get() as conn:"},{"line_number":412,"context_line":"            results \u003d []"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_1fb059dc","line":409,"in_reply_to":"3f79a3b5_4224c7c3","updated":"2018-11-01 17:10:19.000000000","message":"Ha! The whole argument is moot anyway :P\n\nWe decoded everything already in the new else block.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":406,"context_line":"        if delimiter and not prefix:"},{"line_number":407,"context_line":"            prefix \u003d \u0027\u0027"},{"line_number":408,"context_line":"        if prefix:"},{"line_number":409,"context_line":"            end_prefix \u003d prefix[:-1] + chr(ordx(prefix[-1]) + 1)"},{"line_number":410,"context_line":"        orig_marker \u003d marker"},{"line_number":411,"context_line":"        with self.get() as conn:"},{"line_number":412,"context_line":"            results \u003d []"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_4224c7c3","line":409,"in_reply_to":"3f79a3b5_c83e0ae2","updated":"2018-11-01 01:58:14.000000000","message":"That\u0027s nice, but transfers the problem into the casting of the result, because  prefix[:-1] + chr(ordx(prefix[-1:]) + 1)  produces \"TypeError: can\u0027t concat str to bytes\". You know, I actually started by making a function called \"byt()\": same as chr(), but produces bytes. Seemed ordx was better, although same thing really.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"8c8acead0500592b2a146a7fbf6e22278dc9a482","unresolved":false,"context_lines":[{"line_number":398,"context_line":"            if isinstance(prefix, bytes):"},{"line_number":399,"context_line":"                prefix \u003d prefix.decode(\u0027utf-8\u0027)"},{"line_number":400,"context_line":"            if isinstance(delimiter, bytes):"},{"line_number":401,"context_line":"                delimiter \u003d delimiter.decode(\u0027utf-8\u0027)"},{"line_number":402,"context_line":"        if reverse:"},{"line_number":403,"context_line":"            # Reverse the markers if we are reversing the listing."},{"line_number":404,"context_line":"            marker, end_marker \u003d end_marker, marker"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_9b11fd62","line":401,"updated":"2018-11-14 05:01:33.000000000","message":"wow really, man why can\u0027t py2 and py3 be more compatible","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"7cfad57ff912b3b7f2183e4242963fcfc810a4df","unresolved":false,"context_lines":[{"line_number":398,"context_line":"            if isinstance(prefix, bytes):"},{"line_number":399,"context_line":"                prefix \u003d prefix.decode(\u0027utf-8\u0027)"},{"line_number":400,"context_line":"            if isinstance(delimiter, bytes):"},{"line_number":401,"context_line":"                delimiter \u003d delimiter.decode(\u0027utf-8\u0027)"},{"line_number":402,"context_line":"        if reverse:"},{"line_number":403,"context_line":"            # Reverse the markers if we are reversing the listing."},{"line_number":404,"context_line":"            marker, end_marker \u003d end_marker, marker"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_5a30f1b7","line":401,"in_reply_to":"3f79a3b5_9b11fd62","updated":"2018-11-20 04:27:01.000000000","message":"Tim said that we could have a function like utf8encode() above. The question is what to call it. It does the UTF-8 decode, but its nature is not \"utf8decode\". Maybe \"cast_to_native\"? Leaving it inline for now.","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5ee7897acf16f5610295ece911ae489c8063c308","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"import six"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"from swift.common.utils import Timestamp, ordx"},{"line_number":26,"context_line":"from swift.common.db import DatabaseBroker, utf8encode, zero_like"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"DATADIR \u003d \u0027accounts\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_872062fb","line":25,"range":{"start_line":25,"start_character":42,"end_line":25,"end_character":46},"updated":"2018-12-03 23:43:42.000000000","message":"Don\u0027t actually need it -- by the time we\u0027re dealing with prefix/delimiter, they\u0027re always native strings so ord works just fine.","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2ff9a2712d5291408e5852455ad061a511e34b79","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"import six"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"from swift.common.utils import Timestamp, ordx"},{"line_number":26,"context_line":"from swift.common.db import DatabaseBroker, utf8encode, zero_like"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"DATADIR \u003d \u0027accounts\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_54669a29","line":25,"range":{"start_line":25,"start_character":42,"end_line":25,"end_character":46},"in_reply_to":"3f79a3b5_872062fb","updated":"2018-12-15 00:58:51.000000000","message":"Done","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"}],"swift/account/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":178,"context_line":"                    if is_sys_or_user_meta(\u0027account\u0027, key):"},{"line_number":179,"context_line":"                        # This is a bit awkward... We\u0027re calling on broker\u0027s"},{"line_number":180,"context_line":"                        # update_metadata() below, which is perfectly capable"},{"line_number":181,"context_line":"                        # of validating the UTF-8. However, on py3, bytes"},{"line_number":182,"context_line":"                        # cannot be stuffed into JSON and so the update crashes"},{"line_number":183,"context_line":"                        # with valid values. Need to turn it into unicode here"},{"line_number":184,"context_line":"                        # and thus we must catch here, instead of relying"},{"line_number":185,"context_line":"                        # on validation later."}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_e89ae60f","line":182,"range":{"start_line":181,"start_character":68,"end_line":182,"end_character":79},"updated":"2018-10-26 23:30:24.000000000","message":"??\n\nWe\u0027re not talking about the metadata.update() call here, right? When are we trying to serialize JSON with bytes in the dict? Down in broker.update_metadata(), I guess?","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":178,"context_line":"                    if is_sys_or_user_meta(\u0027account\u0027, key):"},{"line_number":179,"context_line":"                        # This is a bit awkward... We\u0027re calling on broker\u0027s"},{"line_number":180,"context_line":"                        # update_metadata() below, which is perfectly capable"},{"line_number":181,"context_line":"                        # of validating the UTF-8. However, on py3, bytes"},{"line_number":182,"context_line":"                        # cannot be stuffed into JSON and so the update crashes"},{"line_number":183,"context_line":"                        # with valid values. Need to turn it into unicode here"},{"line_number":184,"context_line":"                        # and thus we must catch here, instead of relying"},{"line_number":185,"context_line":"                        # on validation later."}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_58c83675","line":182,"range":{"start_line":181,"start_character":68,"end_line":182,"end_character":79},"in_reply_to":"3f79a3b5_e89ae60f","updated":"2018-11-01 01:58:14.000000000","message":"Yes","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":188,"context_line":"                        except UnicodeDecodeError:"},{"line_number":189,"context_line":"                            raise HTTPBadRequest("},{"line_number":190,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":191,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":192,"context_line":"            if metadata:"},{"line_number":193,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":194,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_28c77e13","line":191,"updated":"2018-10-26 23:30:24.000000000","message":"I think we could just use wsgi_to_str and avoid the py2/py3 split code-path...\n\nWhat do header names with UTF-8 look like in the env? Should those get similar treatment?","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":188,"context_line":"                        except UnicodeDecodeError:"},{"line_number":189,"context_line":"                            raise HTTPBadRequest("},{"line_number":190,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":191,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":192,"context_line":"            if metadata:"},{"line_number":193,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":194,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_5876f642","line":191,"in_reply_to":"3f79a3b5_28c77e13","updated":"2018-11-01 01:58:14.000000000","message":"How do you get headers in UTF-8 in our model? They all come from environment as \u0027HTTP_xxxxxx\u0027, therefore they are always WSGI string, are they not?\n\nI could add an isinstance(), I suppose. But there was no test that prompted it.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":188,"context_line":"                        except UnicodeDecodeError:"},{"line_number":189,"context_line":"                            raise HTTPBadRequest("},{"line_number":190,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":191,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":192,"context_line":"            if metadata:"},{"line_number":193,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":194,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_9f17899e","line":191,"in_reply_to":"3f79a3b5_5876f642","updated":"2018-11-01 17:10:19.000000000","message":"Yeah, that was my thought, too -- should be wsgi strings. So I think the whole thing could just be\n\n metadata.update((wsgi_to_str(key), (wsgi_to_str(value), timestamp.internal))\n                 for key, value in req.headers.items()\n                 if is_sys_or_user_meta(\u0027account\u0027, key))\n\n(Fortunately, the sys/user meta checks all work with both wsgi and native strings.)","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":222,"context_line":"        drive, part, account \u003d split_and_validate_path(req, 3)"},{"line_number":223,"context_line":"        prefix \u003d get_param(req, \u0027prefix\u0027)"},{"line_number":224,"context_line":"        delimiter \u003d get_param(req, \u0027delimiter\u0027)"},{"line_number":225,"context_line":"        if delimiter and (len(delimiter) \u003e 1 or ordx(delimiter) \u003e 254):"},{"line_number":226,"context_line":"            # delimiters can be made more flexible later"},{"line_number":227,"context_line":"            return HTTPPreconditionFailed(body\u003d\u0027Bad delimiter\u0027)"},{"line_number":228,"context_line":"        limit \u003d constraints.ACCOUNT_LISTING_LIMIT"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_2863fe1a","line":225,"range":{"start_line":225,"start_character":48,"end_line":225,"end_character":52},"updated":"2018-10-26 23:30:24.000000000","message":"nit: This isn\u0027t strictly necessary, yeah? delimiter will never be a raw int...","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":222,"context_line":"        drive, part, account \u003d split_and_validate_path(req, 3)"},{"line_number":223,"context_line":"        prefix \u003d get_param(req, \u0027prefix\u0027)"},{"line_number":224,"context_line":"        delimiter \u003d get_param(req, \u0027delimiter\u0027)"},{"line_number":225,"context_line":"        if delimiter and (len(delimiter) \u003e 1 or ordx(delimiter) \u003e 254):"},{"line_number":226,"context_line":"            # delimiters can be made more flexible later"},{"line_number":227,"context_line":"            return HTTPPreconditionFailed(body\u003d\u0027Bad delimiter\u0027)"},{"line_number":228,"context_line":"        limit \u003d constraints.ACCOUNT_LISTING_LIMIT"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_386d3a67","line":225,"range":{"start_line":225,"start_character":48,"end_line":225,"end_character":52},"in_reply_to":"3f79a3b5_2863fe1a","updated":"2018-11-01 01:58:14.000000000","message":"What about 0xFF?","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":222,"context_line":"        drive, part, account \u003d split_and_validate_path(req, 3)"},{"line_number":223,"context_line":"        prefix \u003d get_param(req, \u0027prefix\u0027)"},{"line_number":224,"context_line":"        delimiter \u003d get_param(req, \u0027delimiter\u0027)"},{"line_number":225,"context_line":"        if delimiter and (len(delimiter) \u003e 1 or ordx(delimiter) \u003e 254):"},{"line_number":226,"context_line":"            # delimiters can be made more flexible later"},{"line_number":227,"context_line":"            return HTTPPreconditionFailed(body\u003d\u0027Bad delimiter\u0027)"},{"line_number":228,"context_line":"        limit \u003d constraints.ACCOUNT_LISTING_LIMIT"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_9f45499e","line":225,"range":{"start_line":225,"start_character":48,"end_line":225,"end_character":52},"in_reply_to":"3f79a3b5_386d3a67","updated":"2018-11-01 17:10:19.000000000","message":"Two bits:\n\n- the result of get_param should be some type of string; even if it\u0027s a byte string of length 1, it\u0027ll still work with plain old ord()\n- \\xff should get rejected in get_param","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"8c8acead0500592b2a146a7fbf6e22278dc9a482","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                        except UnicodeDecodeError:"},{"line_number":190,"context_line":"                            raise HTTPBadRequest("},{"line_number":191,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":192,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":193,"context_line":"            if metadata:"},{"line_number":194,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":195,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_bb163957","line":192,"updated":"2018-11-14 05:01:33.000000000","message":"So we can\u0027t move this so it becomes the first thing update_metadata does in db.py? because surely it\u0027ll be similar in the container server?","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5ee7897acf16f5610295ece911ae489c8063c308","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                        except UnicodeDecodeError:"},{"line_number":190,"context_line":"                            raise HTTPBadRequest("},{"line_number":191,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":192,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":193,"context_line":"            if metadata:"},{"line_number":194,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":195,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_678666ba","line":192,"in_reply_to":"3f79a3b5_1ac59990","updated":"2018-12-03 23:43:42.000000000","message":"Yeah -- if I need to tweak a container db in a REPL, I\u0027d much *much* prefer that strings be native, not wsgi. Even better is having some affordances for using UTF-8-encoded byte strings on py3 or unicode strings on py2 (which I *think* we already have?)","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"7cfad57ff912b3b7f2183e4242963fcfc810a4df","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                        except UnicodeDecodeError:"},{"line_number":190,"context_line":"                            raise HTTPBadRequest("},{"line_number":191,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":192,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":193,"context_line":"            if metadata:"},{"line_number":194,"context_line":"                broker.update_metadata(metadata, validate_metadata\u003dTrue)"},{"line_number":195,"context_line":"            if created:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_1ac59990","line":192,"in_reply_to":"3f79a3b5_bb163957","updated":"2018-11-20 04:27:01.000000000","message":"Factoring things properly sounds persuasive, but I guess I forgot one thing: the strings here are in WSGI. The update_metadata method cannot assume this: if it sees a unicode string, then it is a unicode string for it (well, for sqlite).\n\nI\u0027m going to update the comment, but leave the code here after all, sorry.\n\nWell, we can always factor it in, but I suspect it\u0027s not going to be as obvious as it seems.","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5ee7897acf16f5610295ece911ae489c8063c308","unresolved":false,"context_lines":[{"line_number":180,"context_line":"                        # updata_metadata eats the data."},{"line_number":181,"context_line":"                        try:"},{"line_number":182,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":183,"context_line":"                        except UnicodeDecodeError:"},{"line_number":184,"context_line":"                            raise HTTPBadRequest("},{"line_number":185,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":186,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_679f46f6","line":183,"range":{"start_line":183,"start_character":31,"end_line":183,"end_character":49},"updated":"2018-12-03 23:43:42.000000000","message":"Right... we don\u0027t want to use wsgi_to_str() here, because it consciously *avoids* ever raising decode errors...","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":182,"context_line":"                        try:"},{"line_number":183,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":184,"context_line":"                        except UnicodeDecodeError:"},{"line_number":185,"context_line":"                            raise HTTPBadRequest("},{"line_number":186,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"},{"line_number":187,"context_line":"                        metadata[key] \u003d (value, timestamp.internal)"},{"line_number":188,"context_line":"            if metadata:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_5a39aa3a","line":185,"updated":"2018-12-19 19:19:32.000000000","message":"I feel like this should\u0027ve already been covered somewhere, but I\u0027m not sure where... over at https://github.com/openstack/swift/blob/2.20.0/swift/proxy/controllers/account.py#L116-L118 maybe?\n\nHmm... What are we doing on py2 if a *direct* client tries to send non-UTF8 data??","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"872d839ba299f2c5448b599d8a70448df4c95012","unresolved":false,"context_lines":[{"line_number":178,"context_line":"                for key, value in req.headers.items():"},{"line_number":179,"context_line":"                    if is_sys_or_user_meta(\u0027account\u0027, key):"},{"line_number":180,"context_line":"                        # Cast to native strings, so that json inside"},{"line_number":181,"context_line":"                        # updata_metadata eats the data."},{"line_number":182,"context_line":"                        try:"},{"line_number":183,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":184,"context_line":"                        except UnicodeDecodeError:"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_7f4a517c","line":181,"updated":"2018-12-19 06:16:16.000000000","message":"s/updata_metadata/update_metadata/","commit_id":"0f505ad968571687181834da3559db6e595a52db"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"ef7922060dd24a6d74ea256b78f2ddc2089b614f","unresolved":false,"context_lines":[{"line_number":178,"context_line":"                for key, value in req.headers.items():"},{"line_number":179,"context_line":"                    if is_sys_or_user_meta(\u0027account\u0027, key):"},{"line_number":180,"context_line":"                        # Cast to native strings, so that json inside"},{"line_number":181,"context_line":"                        # updata_metadata eats the data."},{"line_number":182,"context_line":"                        try:"},{"line_number":183,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":184,"context_line":"                        except UnicodeDecodeError:"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_4e523bdb","line":181,"in_reply_to":"3f79a3b5_7f4a517c","updated":"2018-12-19 17:57:39.000000000","message":"Done","commit_id":"0f505ad968571687181834da3559db6e595a52db"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"872d839ba299f2c5448b599d8a70448df4c95012","unresolved":false,"context_lines":[{"line_number":180,"context_line":"                        # Cast to native strings, so that json inside"},{"line_number":181,"context_line":"                        # updata_metadata eats the data."},{"line_number":182,"context_line":"                        try:"},{"line_number":183,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":184,"context_line":"                        except UnicodeDecodeError:"},{"line_number":185,"context_line":"                            raise HTTPBadRequest("},{"line_number":186,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_1f29dda0","line":183,"updated":"2018-12-19 06:16:16.000000000","message":"This is where I get confused.. and honestly I feel I may always be confused. But if the value contained a utf-8 character that wasn\u0027t in the latin-1 set, this would just fail right? or rather throw a unicodeEncodeError. So what\u0027s rules do we play with now? everything via wsgi should only be latin-1?","commit_id":"0f505ad968571687181834da3559db6e595a52db"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"ef7922060dd24a6d74ea256b78f2ddc2089b614f","unresolved":false,"context_lines":[{"line_number":180,"context_line":"                        # Cast to native strings, so that json inside"},{"line_number":181,"context_line":"                        # updata_metadata eats the data."},{"line_number":182,"context_line":"                        try:"},{"line_number":183,"context_line":"                            value \u003d value.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":184,"context_line":"                        except UnicodeDecodeError:"},{"line_number":185,"context_line":"                            raise HTTPBadRequest("},{"line_number":186,"context_line":"                                \u0027Metadata must be valid UTF-8\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_71f6f62e","line":183,"in_reply_to":"3f79a3b5_1f29dda0","updated":"2018-12-19 17:57:39.000000000","message":"The PEP-333/3333 specification writers made these decisions because while HTTP is all in bytes, and the WSGI framework (Apache or eventlet) cannot know the encoding, they wanted to deliver the string data to WSGI consumer (Swift), so the data could be manipulated as strings. The Solomon solution was to take bytes verbatim and \"decode\" them using the latin-1 as identity encoding.\n\nBut those bytes - in case of Swift - were UTF-8 strings (just WSGI didn\u0027t know that). So, if you \"encode\" those strings with identity back into original HTTP bytes, you get UTF-8 byte strings. Then, you can decode them as normal and get strings. Thus, v.encode(\u0027latin-1\u0027).decode(\u0027utf-8\u0027).\n\nSorry it\u0027s a little too verbose an explanation.","commit_id":"0f505ad968571687181834da3559db6e595a52db"}],"swift/common/db_replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":32,"context_line":"from swift.common.utils import get_logger, whataremyips, storage_directory, \\"},{"line_number":33,"context_line":"    renamer, mkdirs, lock_parent_directory, config_true_value, \\"},{"line_number":34,"context_line":"    unlink_older_than, dump_recon_cache, rsync_module_interpolation, \\"},{"line_number":35,"context_line":"    json, parse_override_options, round_robin_iter, Everything, get_db_files, \\"},{"line_number":36,"context_line":"    parse_db_filename, quote"},{"line_number":37,"context_line":"from swift.common import ring"},{"line_number":38,"context_line":"from swift.common.ring.utils import is_local_device"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_883b3237","side":"PARENT","line":35,"range":{"start_line":35,"start_character":4,"end_line":35,"end_character":8},"updated":"2018-10-26 23:30:24.000000000","message":"++","commit_id":"6bb8229b75de74c11f46de54bbca2d10bfe3bada"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":32,"context_line":"from swift.common.utils import get_logger, whataremyips, storage_directory, \\"},{"line_number":33,"context_line":"    renamer, mkdirs, lock_parent_directory, config_true_value, \\"},{"line_number":34,"context_line":"    unlink_older_than, dump_recon_cache, rsync_module_interpolation, \\"},{"line_number":35,"context_line":"    json, parse_override_options, round_robin_iter, Everything, get_db_files, \\"},{"line_number":36,"context_line":"    parse_db_filename, quote"},{"line_number":37,"context_line":"from swift.common import ring"},{"line_number":38,"context_line":"from swift.common.ring.utils import is_local_device"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_9891ee48","side":"PARENT","line":35,"range":{"start_line":35,"start_character":4,"end_line":35,"end_character":8},"in_reply_to":"3f79a3b5_883b3237","updated":"2018-11-01 01:58:14.000000000","message":"I prefer alphabetical order, but I don\u0027t want to get distracted from the py3 focus.","commit_id":"6bb8229b75de74c11f46de54bbca2d10bfe3bada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5ee7897acf16f5610295ece911ae489c8063c308","unresolved":false,"context_lines":[{"line_number":477,"context_line":"        elif response.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":478,"context_line":"            raise DriveNotMounted()"},{"line_number":479,"context_line":"        elif 200 \u003c\u003d response.status \u003c 300:"},{"line_number":480,"context_line":"            rinfo \u003d json.loads(response.data.decode(\u0027ascii\u0027))"},{"line_number":481,"context_line":"            local_sync \u003d broker.get_sync(rinfo[\u0027id\u0027], incoming\u003dFalse)"},{"line_number":482,"context_line":"            if rinfo.get(\u0027metadata\u0027, \u0027\u0027):"},{"line_number":483,"context_line":"                broker.update_metadata(json.loads(rinfo[\u0027metadata\u0027]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_27486e87","line":480,"range":{"start_line":480,"start_character":45,"end_line":480,"end_character":60},"updated":"2018-12-03 23:43:42.000000000","message":"nit: Not needed.","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2ff9a2712d5291408e5852455ad061a511e34b79","unresolved":false,"context_lines":[{"line_number":477,"context_line":"        elif response.status \u003d\u003d HTTP_INSUFFICIENT_STORAGE:"},{"line_number":478,"context_line":"            raise DriveNotMounted()"},{"line_number":479,"context_line":"        elif 200 \u003c\u003d response.status \u003c 300:"},{"line_number":480,"context_line":"            rinfo \u003d json.loads(response.data.decode(\u0027ascii\u0027))"},{"line_number":481,"context_line":"            local_sync \u003d broker.get_sync(rinfo[\u0027id\u0027], incoming\u003dFalse)"},{"line_number":482,"context_line":"            if rinfo.get(\u0027metadata\u0027, \u0027\u0027):"},{"line_number":483,"context_line":"                broker.update_metadata(json.loads(rinfo[\u0027metadata\u0027]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_146822ef","line":480,"range":{"start_line":480,"start_character":45,"end_line":480,"end_character":60},"in_reply_to":"3f79a3b5_27486e87","updated":"2018-12-15 00:58:51.000000000","message":"done","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"}],"swift/common/request_helpers.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":71,"context_line":"    else:"},{"line_number":72,"context_line":"        if value:"},{"line_number":73,"context_line":"            try:"},{"line_number":74,"context_line":"                if isinstance(value, six.text_type):"},{"line_number":75,"context_line":"                    value \u003d value.encode(\u0027latin1\u0027)"},{"line_number":76,"context_line":"            except UnicodeEncodeError:"},{"line_number":77,"context_line":"                # This happens, remarkably enough, when WSGI string is not"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_da65ba1c","line":74,"range":{"start_line":74,"start_character":16,"end_line":74,"end_character":51},"updated":"2018-12-19 19:19:32.000000000","message":"Would this ever *not* be the case?","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":74,"context_line":"                if isinstance(value, six.text_type):"},{"line_number":75,"context_line":"                    value \u003d value.encode(\u0027latin1\u0027)"},{"line_number":76,"context_line":"            except UnicodeEncodeError:"},{"line_number":77,"context_line":"                # This happens, remarkably enough, when WSGI string is not"},{"line_number":78,"context_line":"                # a valid latin-1, and passed through parse_qsl()."},{"line_number":79,"context_line":"                raise HTTPBadRequest("},{"line_number":80,"context_line":"                    request\u003dreq, content_type\u003d\u0027text/plain\u0027,"},{"line_number":81,"context_line":"                    body\u003d\u0027\"%s\" parameter not valid UTF-8\u0027 % name)"},{"line_number":82,"context_line":"            try:"},{"line_number":83,"context_line":"                # Ensure UTF8ness since we\u0027re at it"},{"line_number":84,"context_line":"                value \u003d value.decode(\u0027utf8\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_7aa90624","line":81,"range":{"start_line":77,"start_character":16,"end_line":81,"end_character":65},"updated":"2018-12-19 19:19:32.000000000","message":"Things like this are what make me want to have Request.params just emit WSGI strings from the get-go, without trying to lossily round-trip through UTF-8.","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"}],"swift/common/swob.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":766,"context_line":"            if is_wsgi_string_field:"},{"line_number":767,"context_line":"                # Check that input is valid before setting"},{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_78f0e7da","line":769,"range":{"start_line":769,"start_character":42,"end_line":769,"end_character":58},"updated":"2018-10-26 23:30:24.000000000","message":"This is a tighter check than we\u0027ve had previously...","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":766,"context_line":"            if is_wsgi_string_field:"},{"line_number":767,"context_line":"                # Check that input is valid before setting"},{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_9fda29e4","line":769,"range":{"start_line":769,"start_character":42,"end_line":769,"end_character":58},"in_reply_to":"3f79a3b5_38a21a99","updated":"2018-11-01 17:10:19.000000000","message":"Haven\u0027t decided yet :-) Still thinking.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":766,"context_line":"            if is_wsgi_string_field:"},{"line_number":767,"context_line":"                # Check that input is valid before setting"},{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_38a21a99","line":769,"range":{"start_line":769,"start_character":42,"end_line":769,"end_character":58},"in_reply_to":"3f79a3b5_78f0e7da","updated":"2018-11-01 01:58:14.000000000","message":"Indeed... Well, we didn\u0027t have anything before, and I thought that you could have gone further. Is this wrong?","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"    return property(getter, setter, doc\u003d(\"Get and set the %s property \""}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_b823df56","line":771,"updated":"2018-10-26 23:30:24.000000000","message":"So if we get bytes on py3 and is_wsgi_string_field\u003dFalse, we store bytes in the environ?\n\nI suppose it shouldn\u0027t really affect environ[\u0027wsgi.input\u0027], but we might need to keep this in mind for environ[\u0027swob.ACL\u0027]...","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"    return property(getter, setter, doc\u003d(\"Get and set the %s property \""}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_dfa74151","line":771,"in_reply_to":"3f79a3b5_98aa4e6b","updated":"2018-11-01 17:10:19.000000000","message":"\u003e I really don\u0027t want to circle back to swob every time we adapt new module to py3\n\n++\n\nPart of the reason I keep pushing back on changes to swob is that I thought we already had things to the point that this wouldn\u0027t be necessary.\n\nYou\u0027re right, of course, that there\u0027s no saving people who start poking things into env directly -- but I worry a bit about making it \"easy\" to do that with our helper properties.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"    return property(getter, setter, doc\u003d(\"Get and set the %s property \""}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_98aa4e6b","line":771,"in_reply_to":"3f79a3b5_b823df56","updated":"2018-11-01 01:58:14.000000000","message":"At first I thought it was appropriate, in that if someone screws with environ (yes, by putting something like function pointers in there), then I\u0027m not there to damage it. Truth to be told, I really don\u0027t want to circle back to swob every time we adapt new module to py3. Surely we can do it right eventually.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":751,"context_line":"def _req_environ_property(environ_field, is_wsgi_string_field\u003dTrue):"},{"line_number":752,"context_line":"    \"\"\""},{"line_number":753,"context_line":"    Set and retrieve value of the environ_field entry in self.environ."},{"line_number":754,"context_line":"    (Used by both request and response)"},{"line_number":755,"context_line":"    \"\"\""},{"line_number":756,"context_line":"    def getter(self):"},{"line_number":757,"context_line":"        return self.environ.get(environ_field, None)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_758fd776","line":754,"range":{"start_line":754,"start_character":30,"end_line":754,"end_character":38},"updated":"2018-12-19 19:19:32.000000000","message":"Off-topic: Is that right? I don\u0027t see it being used anywhere in Response...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":766,"context_line":"            if is_wsgi_string_field:"},{"line_number":767,"context_line":"                # Check that input is valid before setting"},{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_d5682bbc","line":769,"range":{"start_line":769,"start_character":42,"end_line":769,"end_character":58},"updated":"2018-12-19 19:19:32.000000000","message":"I don\u0027t know that we necessarily expect that `value` will be UTF-8-compatible :-(\n\nApparently, user metadata in objects can be arbitrary byte strings, though I suppose that doesn\u0027t really affect _req_environ_property. But I\u0027d also worry about clients trying to cause an exception by sending wonky Referer or User-Agent headers...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":767,"context_line":"                # Check that input is valid before setting"},{"line_number":768,"context_line":"                if isinstance(value, str):"},{"line_number":769,"context_line":"                    value.encode(\u0027latin1\u0027).decode(\u0027utf-8\u0027)"},{"line_number":770,"context_line":"                if isinstance(value, bytes):"},{"line_number":771,"context_line":"                    value \u003d value.decode(\u0027latin1\u0027)"},{"line_number":772,"context_line":"            self.environ[environ_field] \u003d value"},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"    return property(getter, setter, doc\u003d(\"Get and set the %s property \""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_35615fa4","line":771,"range":{"start_line":770,"start_character":16,"end_line":771,"end_character":50},"updated":"2018-12-19 19:19:32.000000000","message":"Should this be de-dented, or was it intentional to have it only apply when is_wsgi_string_field\u003dTrue? Maybe it\u0027s moot, since it\u0027d only make a difference for wsgi.input (which should never be set to bytes *or* unicode) and swob.ACL (which... should be coming from headers or memcache? so... probably already... some form of unicode?)\n\n*shrug* probably doesn\u0027t matter... or if it does, we\u0027ll fix it later.","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a748291425d6483b0b11e6f1ba1b5fcc94c412b1","unresolved":false,"context_lines":[{"line_number":953,"context_line":"        if self._params_cache is None:"},{"line_number":954,"context_line":"            if \u0027QUERY_STRING\u0027 in self.environ:"},{"line_number":955,"context_line":"                self._params_cache \u003d dict("},{"line_number":956,"context_line":"                    urllib.parse.parse_qsl(self.environ[\u0027QUERY_STRING\u0027], True))"},{"line_number":957,"context_line":"            else:"},{"line_number":958,"context_line":"                self._params_cache \u003d {}"},{"line_number":959,"context_line":"        return self._params_cache"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_b787f05c","line":956,"updated":"2018-12-15 01:24:06.000000000","message":"OK... so we\u0027ll continue decoding quoted values as UTF-8, and swap out for \\ufffd if there\u0027d anything invalid.\n\nI think there\u0027s still a landmine here. QUERY_STRING is (must be, according to spec) a WSGI string, so if someone sent a request line like\n\n b\u0027GET /v1/a/c?prefix\u003d\\xe1\\x88\\xb4/%E4%8C%A1/ HTTP/1.1\u0027\n\nwe\u0027ll have a QUERY_STRING like\n\n u\u0027prefix\u003d\\u00e1\\u0088\\u00b4/%E4%8C%A1/\u0027\n\nand parse_qsl will give us\n\n [(u\u0027prefix\u0027, u\u0027\\u00e1\\u0088\\u00b4/\\u4321/\u0027)]\n\nwhereas on py2, we\u0027d get\n\n b\u0027prefix\u003d\\xe1\\x88\\xb4/%E4%8C%A1/\u0027\n\nand parse_qsl will give us\n\n [(b\u0027prefix\u0027, b\u0027\\xe1\\x88\\xb4/\\xe4\\x8c\\xa1/\u0027)]\n\nwhich would UTF8-decode to\n\n [(u\u0027prefix\u0027, u\u0027\\u1234/\\u4321/\u0027)]","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"0ce1b568ba9b962be09ead89a99a6779ed5b0edd","unresolved":false,"context_lines":[{"line_number":953,"context_line":"        if self._params_cache is None:"},{"line_number":954,"context_line":"            if \u0027QUERY_STRING\u0027 in self.environ:"},{"line_number":955,"context_line":"                self._params_cache \u003d dict("},{"line_number":956,"context_line":"                    urllib.parse.parse_qsl(self.environ[\u0027QUERY_STRING\u0027], True))"},{"line_number":957,"context_line":"            else:"},{"line_number":958,"context_line":"                self._params_cache \u003d {}"},{"line_number":959,"context_line":"        return self._params_cache"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_669e6c53","line":956,"in_reply_to":"3f79a3b5_b787f05c","updated":"2018-12-18 02:07:16.000000000","message":"I don\u0027t understand what you want from me. My previous solution provided a consistent result, like on py2. The price for that was the UTF-8 product, and a bit of back-and-forth encoding. But you had to insist on letting the WSGI enconding through and yes, this is the result. The only way to obtain the consistent presentation in WSIG is first use my old cumbersome code to get UTF-8 and then latin-1 it into WSGI :-)  You cannot persuade parse_qs to do the right thing otherwise!\n\nI think we need to go back to what it was.","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":958,"context_line":"                else:"},{"line_number":959,"context_line":"                    self._params_cache \u003d dict(urllib.parse.parse_qsl("},{"line_number":960,"context_line":"                        self.environ[\u0027QUERY_STRING\u0027],"},{"line_number":961,"context_line":"                        keep_blank_values\u003dTrue, encoding\u003d\u0027latin-1\u0027))"},{"line_number":962,"context_line":"            else:"},{"line_number":963,"context_line":"                self._params_cache \u003d {}"},{"line_number":964,"context_line":"        return self._params_cache"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_dcf281e1","line":961,"updated":"2018-12-19 19:19:32.000000000","message":"Yes! OMG yes. I was just about to write something like\n\nFWIW, I don\u0027t think I had any big objections to the changes to Request.params -- I was mainly worried about the implications of the _req_environ_property changes (at least, as far as swob is concerned). I guess you\u0027re thinking of when I said\n\n\u003e and what do you think about having Request.params populated with WSGI strings? agree that we can\u0027t keep decoding as utf8, but that seems like a not-crazy middle ground? probably need to fix up the setter, too...\n\nin IRC? I just meant to leave it as a WSGI string instead of going to bytes. Like,\n\n if six.PY2:\n     self._params_cache \u003d dict(urllib.parse.parse_qsl(\n         self.environ[\u0027QUERY_STRING\u0027], True))\n else:\n     self._params_cache \u003d dict(urllib.parse.parse_qsl(\n         self.environ[\u0027QUERY_STRING\u0027], True, encoding\u003d\u0027latin-1\u0027))","commit_id":"0f505ad968571687181834da3559db6e595a52db"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c68dfb8f425b4d50074a10855f51e1c8bba8e3b6","unresolved":false,"context_lines":[{"line_number":958,"context_line":"                else:"},{"line_number":959,"context_line":"                    self._params_cache \u003d dict(urllib.parse.parse_qsl("},{"line_number":960,"context_line":"                        self.environ[\u0027QUERY_STRING\u0027],"},{"line_number":961,"context_line":"                        keep_blank_values\u003dTrue, encoding\u003d\u0027latin-1\u0027))"},{"line_number":962,"context_line":"            else:"},{"line_number":963,"context_line":"                self._params_cache \u003d {}"},{"line_number":964,"context_line":"        return self._params_cache"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_f23441d9","line":961,"in_reply_to":"3f79a3b5_dcf281e1","updated":"2018-12-20 00:03:13.000000000","message":"I see, I miscounted arguments. I thought keep_blank_values moved, so it could not remain positional.","commit_id":"0f505ad968571687181834da3559db6e595a52db"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":3517,"context_line":"    return []"},{"line_number":3518,"context_line":""},{"line_number":3519,"context_line":""},{"line_number":3520,"context_line":"def streq_const_time(s1, s2):"},{"line_number":3521,"context_line":"    \"\"\"Constant-time string comparison."},{"line_number":3522,"context_line":""},{"line_number":3523,"context_line":"    :params s1: the first string"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_28069e86","line":3520,"updated":"2018-10-26 23:30:24.000000000","message":"Off-topic: Do we have any constraints on which versions of py27 we support? If we require python\u003e\u003d2.7.7, we could swap this out for https://docs.python.org/2/library/hmac.html#hmac.compare_digest","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":3517,"context_line":"    return []"},{"line_number":3518,"context_line":""},{"line_number":3519,"context_line":""},{"line_number":3520,"context_line":"def streq_const_time(s1, s2):"},{"line_number":3521,"context_line":"    \"\"\"Constant-time string comparison."},{"line_number":3522,"context_line":""},{"line_number":3523,"context_line":"    :params s1: the first string"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_f8da621a","line":3520,"in_reply_to":"3f79a3b5_28069e86","updated":"2018-11-01 01:58:14.000000000","message":"We\u0027re still on python-2.7.5 in RHEL-7.6 even.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":5601,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5602,"context_line":"# values of ASCII-encoded charactes and bytes. Has to be used, because of tests"},{"line_number":5603,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5604,"context_line":"def ordx(s):"},{"line_number":5605,"context_line":"    if six.PY2:"},{"line_number":5606,"context_line":"        # We don\u0027t need to check for len(s)\u003d\u003d1, because the built-in does."},{"line_number":5607,"context_line":"        return ord(s)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_db1d7261","line":5604,"updated":"2018-10-26 23:30:24.000000000","message":"I want to feel this out a bit more -- it\u0027s not as clear of a win for me as the wsgi_to_* and *_to_wsgi helpers.\n\nGiven that (most?) callers of ord() would need to be updated anyway, how much worse would it be to switch to slicing the byte string (which produces another byte string) where we\u0027re currently grabbing a singular index (thus getting an int)?","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":5601,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5602,"context_line":"# values of ASCII-encoded charactes and bytes. Has to be used, because of tests"},{"line_number":5603,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5604,"context_line":"def ordx(s):"},{"line_number":5605,"context_line":"    if six.PY2:"},{"line_number":5606,"context_line":"        # We don\u0027t need to check for len(s)\u003d\u003d1, because the built-in does."},{"line_number":5607,"context_line":"        return ord(s)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_78c65244","line":5604,"in_reply_to":"3f79a3b5_db1d7261","updated":"2018-11-01 01:58:14.000000000","message":"Answered this comment at the call site - when I tried that, I ended with byt(). That said, I think a bulk helper is better, because calling non-native functions is expensive in Python. Only in this case we\u0027re not cycling doing ordx(). So it was easier to ensure correctness.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":5617,"context_line":"    if six.PY3:"},{"line_number":5618,"context_line":"        if isinstance(s, bytes):"},{"line_number":5619,"context_line":"            return json.loads(s.decode(\u0027utf-8\u0027))"},{"line_number":5620,"context_line":"    return json.loads(s)"},{"line_number":5621,"context_line":""},{"line_number":5622,"context_line":""},{"line_number":5623,"context_line":"# Same as json_loads, only for wsgi.input or other file-like."}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_8824920f","line":5620,"updated":"2018-10-26 23:30:24.000000000","message":"I\u0027m torn between wanting to either\n\n- force us to think about encodings at the point at which we\u0027re reading things off the wire or\n- just monkey-patch stdlib on py35...\n\nThe helper is nice, I guess, but I feel like it\u0027ll always have spotty usage, partly because I already started littering our code with .decode(\u0027ascii\u0027) and partly because stdlib is the more-obvious tool to reach for :-/","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edfbffa0a57241ff7230704bc312272e276b4732","unresolved":false,"context_lines":[{"line_number":5617,"context_line":"    if six.PY3:"},{"line_number":5618,"context_line":"        if isinstance(s, bytes):"},{"line_number":5619,"context_line":"            return json.loads(s.decode(\u0027utf-8\u0027))"},{"line_number":5620,"context_line":"    return json.loads(s)"},{"line_number":5621,"context_line":""},{"line_number":5622,"context_line":""},{"line_number":5623,"context_line":"# Same as json_loads, only for wsgi.input or other file-like."}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_91f8ddc1","line":5620,"in_reply_to":"3f79a3b5_38d0daf4","updated":"2018-11-01 17:10:19.000000000","message":"OK, thinking more in terms of long-term maintainability, having just one place to work around the bad, and being able to eventually remove the hack while leaving minimal scarring in the code... I\u0027m kinda coming around on something like http://paste.openstack.org/show/733782/\n\nBonus: we get json.load for free because (as you noted) it just uses json.loads under the covers.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"aae1d10c09729ca5c6e0c8e3bc073bd9edda187f","unresolved":false,"context_lines":[{"line_number":5617,"context_line":"    if six.PY3:"},{"line_number":5618,"context_line":"        if isinstance(s, bytes):"},{"line_number":5619,"context_line":"            return json.loads(s.decode(\u0027utf-8\u0027))"},{"line_number":5620,"context_line":"    return json.loads(s)"},{"line_number":5621,"context_line":""},{"line_number":5622,"context_line":""},{"line_number":5623,"context_line":"# Same as json_loads, only for wsgi.input or other file-like."}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_38d0daf4","line":5620,"in_reply_to":"3f79a3b5_8824920f","updated":"2018-11-01 01:58:14.000000000","message":"We have enough thought, but it\u0027s an iterator. I cannot select to decode or not ahead of time. Well, I guess if I know that the iterator is a network socket, then I can.\n\nWe have a decent history disposing of baggage like this, as long as it\u0027s delineated well. The py26 and eventlet-1.10 stuff went away. So I\u0027m not too concerned about technical baggage here.","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"8c8acead0500592b2a146a7fbf6e22278dc9a482","unresolved":false,"context_lines":[{"line_number":5611,"context_line":""},{"line_number":5612,"context_line":""},{"line_number":5613,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5614,"context_line":"# values of ASCII-encoded charactes and bytes. Has to be used, because of tests"},{"line_number":5615,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5616,"context_line":"def ordx(s):"},{"line_number":5617,"context_line":"    if six.PY2:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_5b0b652c","line":5614,"updated":"2018-11-14 05:01:33.000000000","message":"s/charactes/characters/","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"7cfad57ff912b3b7f2183e4242963fcfc810a4df","unresolved":false,"context_lines":[{"line_number":5611,"context_line":""},{"line_number":5612,"context_line":""},{"line_number":5613,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5614,"context_line":"# values of ASCII-encoded charactes and bytes. Has to be used, because of tests"},{"line_number":5615,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5616,"context_line":"def ordx(s):"},{"line_number":5617,"context_line":"    if six.PY2:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_495b5588","line":5614,"in_reply_to":"3f79a3b5_5b0b652c","updated":"2018-11-20 04:27:01.000000000","message":"Done","commit_id":"0588551b03f32592f2d7c7409b94b59dcf2f0383"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5ee7897acf16f5610295ece911ae489c8063c308","unresolved":false,"context_lines":[{"line_number":5623,"context_line":""},{"line_number":5624,"context_line":""},{"line_number":5625,"context_line":"# We only have json.loads() patched on py35, so this helps json.load()."},{"line_number":5626,"context_line":"# In case anyone have concerns about fp.read(), this actually is exactly"},{"line_number":5627,"context_line":"# what the implementation of the stock json.load() does."},{"line_number":5628,"context_line":"def json_load(fp):"},{"line_number":5629,"context_line":"    if six.PY3:"},{"line_number":5630,"context_line":"        return json.loads(fp.read().decode(\u0027utf-8\u0027))"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_876f02fa","line":5627,"range":{"start_line":5626,"start_character":48,"end_line":5627,"end_character":55},"updated":"2018-12-03 23:43:42.000000000","message":"This comment is spot-on, which is why we don\u0027t actually need a json_load helper at all :-)","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2ff9a2712d5291408e5852455ad061a511e34b79","unresolved":false,"context_lines":[{"line_number":5623,"context_line":""},{"line_number":5624,"context_line":""},{"line_number":5625,"context_line":"# We only have json.loads() patched on py35, so this helps json.load()."},{"line_number":5626,"context_line":"# In case anyone have concerns about fp.read(), this actually is exactly"},{"line_number":5627,"context_line":"# what the implementation of the stock json.load() does."},{"line_number":5628,"context_line":"def json_load(fp):"},{"line_number":5629,"context_line":"    if six.PY3:"},{"line_number":5630,"context_line":"        return json.loads(fp.read().decode(\u0027utf-8\u0027))"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_b4760ed6","line":5627,"range":{"start_line":5626,"start_character":48,"end_line":5627,"end_character":55},"in_reply_to":"3f79a3b5_876f02fa","updated":"2018-12-15 00:58:51.000000000","message":"Done","commit_id":"ece1a82b29e1fc4562b79bd837614f06de2636d2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a748291425d6483b0b11e6f1ba1b5fcc94c412b1","unresolved":false,"context_lines":[{"line_number":5632,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5633,"context_line":"# values of ASCII-encoded characters and bytes. This exists because of tests"},{"line_number":5634,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5635,"context_line":"def ordx(s):"},{"line_number":5636,"context_line":"    if six.PY2:"},{"line_number":5637,"context_line":"        # We don\u0027t need to check for len(s)\u003d\u003d1, because the built-in does."},{"line_number":5638,"context_line":"        return ord(s)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_d782ac4b","line":5635,"updated":"2018-12-15 01:24:06.000000000","message":"Can we scrap this now? We only use it up above, and I\u0027m pretty sure that worked before, too...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"ba8f0b138a17a334d2a6618db0fdbad1cc25c833","unresolved":false,"context_lines":[{"line_number":5632,"context_line":"# This re-implementation of py2 ord() is strictly for comparisons of numerical"},{"line_number":5633,"context_line":"# values of ASCII-encoded characters and bytes. This exists because of tests"},{"line_number":5634,"context_line":"# of checksums and ordering of delimeters (they are 1-character long in Swift)."},{"line_number":5635,"context_line":"def ordx(s):"},{"line_number":5636,"context_line":"    if six.PY2:"},{"line_number":5637,"context_line":"        # We don\u0027t need to check for len(s)\u003d\u003d1, because the built-in does."},{"line_number":5638,"context_line":"        return ord(s)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_d745cc14","line":5635,"in_reply_to":"3f79a3b5_d782ac4b","updated":"2018-12-15 01:41:29.000000000","message":"I think it makes test_backend.py nicer, too. We coded it inline previously, essentially.","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"}],"test/unit/account/test_backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":729,"context_line":"               Timestamp(2).internal, Timestamp(0).internal, 0, 0))"},{"line_number":730,"context_line":"        hashb \u003d hashlib.md5(text.encode(\u0027ascii\u0027)).digest()"},{"line_number":731,"context_line":"        hashc \u003d \u0027\u0027.join((\u0027%02x\u0027 % (ordx(a) ^ ordx(b))"},{"line_number":732,"context_line":"                         for a, b in zip(hasha, hashb)))"},{"line_number":733,"context_line":"        self.assertEqual(broker.get_info()[\u0027hash\u0027], hashc)"},{"line_number":734,"context_line":"        broker.put_container(\u0027b\u0027, Timestamp(3).internal,"},{"line_number":735,"context_line":"                             Timestamp(0).internal, 0, 0,"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_158c632b","line":732,"updated":"2018-12-19 19:19:32.000000000","message":"Just a thought, as an alternative to adding a new utility function we\u0027d have to maintain, if the six.PY2 conditional is really irksome:\n\n hashc \u003d \u0027\u0027.join(\u0027%02x\u0027 % (a ^ b)\n                 for a, b in zip(bytearray(hasha),\n                                 bytearray(hashb))\n\n.digest() is going to come out bytes on both py2 and py3, and bytearray has the same sequence-of-ints iterator behavior on both...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"}],"test/unit/account/test_server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":916,"context_line":"        resp \u003d req.get_response(self.controller)"},{"line_number":917,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":918,"context_line":"        self.assertEqual(resp.body.decode(\u0027utf-8\u0027).strip().split(\u0027\\n\u0027),"},{"line_number":919,"context_line":"                         [\u0027c1\u0027, \u0027c2\u0027])"},{"line_number":920,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/a/c1\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":921,"context_line":"                            headers\u003d{\u0027X-Put-Timestamp\u0027: \u00271\u0027,"},{"line_number":922,"context_line":"                                     \u0027X-Delete-Timestamp\u0027: \u00270\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_1528e3ea","line":919,"updated":"2018-12-19 19:19:32.000000000","message":"I woulda been tempted to sprinkle a few more b\u0027s around rather than rely on py2\u0027s b\u0027c1\u0027 \u003d\u003d u\u0027c1\u0027 behavior, but w/e","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":1615,"context_line":"        outbuf \u003d StringIO()"},{"line_number":1616,"context_line":""},{"line_number":1617,"context_line":"        def start_response(*args):"},{"line_number":1618,"context_line":"            outbuf.write(args[0])"},{"line_number":1619,"context_line":""},{"line_number":1620,"context_line":"        self.controller.__call__({\u0027REQUEST_METHOD\u0027: \u0027GET\u0027,"},{"line_number":1621,"context_line":"                                  \u0027SCRIPT_NAME\u0027: \u0027\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_751fd7b8","line":1618,"updated":"2018-12-19 19:19:32.000000000","message":"No more header capture? Meh... not like we used it before, anyway...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":1843,"context_line":"             \u0027replication_server\u0027: \u0027false\u0027})"},{"line_number":1844,"context_line":""},{"line_number":1845,"context_line":"        def start_response(*args):"},{"line_number":1846,"context_line":"            outbuf.write(args[0])"},{"line_number":1847,"context_line":""},{"line_number":1848,"context_line":"        method \u003d \u0027PUT\u0027"},{"line_number":1849,"context_line":"        env \u003d {\u0027REQUEST_METHOD\u0027: method,"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_95cf1325","line":1846,"range":{"start_line":1846,"start_character":12,"end_line":1846,"end_character":18},"updated":"2018-12-19 19:19:32.000000000","message":"Off-topic: are we even using this capture?","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1eb1f30544befbba52a25a6eb16b94b4682c24df","unresolved":false,"context_lines":[{"line_number":1880,"context_line":"             \u0027replication_server\u0027: \u0027false\u0027})"},{"line_number":1881,"context_line":""},{"line_number":1882,"context_line":"        def start_response(*args):"},{"line_number":1883,"context_line":"            outbuf.write(args[0])"},{"line_number":1884,"context_line":""},{"line_number":1885,"context_line":"        method \u003d \u0027PUT\u0027"},{"line_number":1886,"context_line":"        env \u003d {\u0027REQUEST_METHOD\u0027: method,"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_954ab3ab","line":1883,"range":{"start_line":1883,"start_character":12,"end_line":1883,"end_character":18},"updated":"2018-12-19 19:19:32.000000000","message":"...or this one...","commit_id":"bd42d1fc2b6d70178511313b43d3a3a4e09373e8"}],"tox.ini":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a73dfa9cbf54c4a47c0ca2dfbdbe1031e4b055e0","unresolved":false,"context_lines":[{"line_number":29,"context_line":"[testenv:py35]"},{"line_number":30,"context_line":"commands \u003d"},{"line_number":31,"context_line":"  nosetests {posargs:\\"},{"line_number":32,"context_line":"      test/unit/account \\"},{"line_number":33,"context_line":"      test/unit/cli/test_dispersion_report.py \\"},{"line_number":34,"context_line":"      test/unit/cli/test_form_signature.py \\"},{"line_number":35,"context_line":"      test/unit/cli/test_info.py \\"}],"source_content_type":"text/x-properties","patch_set":2,"id":"3f79a3b5_9b337af7","line":32,"updated":"2018-10-26 23:30:24.000000000","message":"Whooo! Consolidation!","commit_id":"f25bae9219d36a5795978d6669366e360db9f772"}]}
