)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d3e30614a5aff8fedc2fb12c948f476bacbc7198","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This patch is expected to start a discussion, and some things will"},{"line_number":20,"context_line":"need investigation (e.g. the etag of an SLO is currently the md5sum"},{"line_number":21,"context_line":"of the concantenated etags of the objects it contains ignoring"},{"line_number":22,"context_line":"the preamble and postamble information)."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"The patch also includes a number of TODO items (it was originally"},{"line_number":25,"context_line":"developed against swift 2.2.x and the tests are currently"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"da6895a0_ff56c5d0","line":22,"range":{"start_line":21,"start_character":54,"end_line":22,"end_character":39},"updated":"2016-10-27 07:06:40.000000000","message":"This part definitely makes me nervous, although I haven\u0027t yet thought about how to address it. This very much reminds me of the MD5 changes necessary around https://github.com/openstack/swift/commit/25d5e68#diff-faee22e330c0eb1272b3b30a9822bc3dL648 -- that may be a reasonable starting model.","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"6e29bf88be90157d6665a03d2344b139cb886d30","unresolved":false,"context_lines":[{"line_number":19,"context_line":"This patch is expected to start a discussion, and some things will"},{"line_number":20,"context_line":"need investigation (e.g. the etag of an SLO is currently the md5sum"},{"line_number":21,"context_line":"of the concantenated etags of the objects it contains ignoring"},{"line_number":22,"context_line":"the preamble and postamble information)."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"The patch also includes a number of TODO items (it was originally"},{"line_number":25,"context_line":"developed against swift 2.2.x and the tests are currently"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"ba5da102_ed681cbd","line":22,"in_reply_to":"ba5da102_89124561","updated":"2016-10-28 08:14:57.000000000","message":"IMO, chuck the literal pieces in the MD5 hasher along with the segment etags.\n\nWe know the rule is \"change a byte, change the etag\", and adding the pre/post-ambles to the segment etags certainly follows the rule. I\u0027m sure there\u0027s a bunch of other ways to do it as well, but this one has the advantage of being easy to code up. :)","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"29be545c47597e03ef38dc904518d7f7dd34a791","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This patch is expected to start a discussion, and some things will"},{"line_number":20,"context_line":"need investigation (e.g. the etag of an SLO is currently the md5sum"},{"line_number":21,"context_line":"of the concantenated etags of the objects it contains ignoring"},{"line_number":22,"context_line":"the preamble and postamble information)."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"The patch also includes a number of TODO items (it was originally"},{"line_number":25,"context_line":"developed against swift 2.2.x and the tests are currently"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"ba5da102_89124561","line":22,"range":{"start_line":21,"start_character":54,"end_line":22,"end_character":39},"in_reply_to":"da6895a0_ff56c5d0","updated":"2016-10-27 14:04:51.000000000","message":"I totally agree, part of the discussion I want to generate from this patch is \u0027what is the correct thing to do for the etag?\u0027","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"519f365d4fa38c8d7a92356ebad0c66cf6ecd26a","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Tim Burke \u003ctim.burke@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2017-10-17 19:22:22 +0000"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add Preamble and Postamble to SLO and SegmentedIterable"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This patch updates the SLO middleware and SegmentedIterable to add"},{"line_number":10,"context_line":"support for user specified preamble and postamble data for each"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":31,"id":"5f4e5783_63e383a5","line":7,"range":{"start_line":7,"start_character":4,"end_line":7,"end_character":26},"updated":"2017-10-17 19:26:32.000000000","message":"Bah, forgot to update the commit message","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0b681f907eaf1a5c04e9226be59709f7958b4fc8","unresolved":false,"context_lines":[{"line_number":11,"context_line":"expected to be base64-encoded data to be added before/after an"},{"line_number":12,"context_line":"object-backed segment within an SLO. To accommodate the potential"},{"line_number":13,"context_line":"extra data we increase the default SLO maximum manifest size from"},{"line_number":14,"context_line":"2MiB to 8MiB. The default maximum number of segments remains 1000,"},{"line_number":15,"context_line":"but this will be enforced separately for object-backed vs."},{"line_number":16,"context_line":"inlined-data segments."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This patch is a prerequisite for a future patch enabling the"},{"line_number":19,"context_line":"download of large objects as tarballs. The TLO patch will be added"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"3f4b6375_eeba65dc","line":16,"range":{"start_line":14,"start_character":14,"end_line":16,"end_character":22},"updated":"2017-10-19 18:05:12.000000000","message":"I *think* this is a sane approach to take? Maybe it would be simply to think about if I just doubled the default max-segments, but without something like https://review.openstack.org/#/c/509306/ I\u0027m a bit hesitant to increase the number of allowable object-backed segments.","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"24aa8acab3c94b07f717afe5736eefd63bf9246d","unresolved":false,"context_lines":[{"line_number":11,"context_line":"expected to be base64-encoded data to be added before/after an"},{"line_number":12,"context_line":"object-backed segment within an SLO. To accommodate the potential"},{"line_number":13,"context_line":"extra data we increase the default SLO maximum manifest size from"},{"line_number":14,"context_line":"2MiB to 8MiB. The default maximum number of segments remains 1000,"},{"line_number":15,"context_line":"but this will be enforced separately for object-backed vs."},{"line_number":16,"context_line":"inlined-data segments."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This patch is a prerequisite for a future patch enabling the"},{"line_number":19,"context_line":"download of large objects as tarballs. The TLO patch will be added"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"3f4b6375_8c0df63f","line":16,"range":{"start_line":14,"start_character":14,"end_line":16,"end_character":22},"in_reply_to":"3f4b6375_6d1be61a","updated":"2017-11-01 21:23:09.000000000","message":"Yeah, I\u0027d proposed https://review.openstack.org/#/c/437173/ but it probably doesn\u0027t make sense to do until we have heartbeating from https://review.openstack.org/#/c/509306/","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"6a4164c0f9fea2f401cf7d888f3a10f6938793bc","unresolved":false,"context_lines":[{"line_number":11,"context_line":"expected to be base64-encoded data to be added before/after an"},{"line_number":12,"context_line":"object-backed segment within an SLO. To accommodate the potential"},{"line_number":13,"context_line":"extra data we increase the default SLO maximum manifest size from"},{"line_number":14,"context_line":"2MiB to 8MiB. The default maximum number of segments remains 1000,"},{"line_number":15,"context_line":"but this will be enforced separately for object-backed vs."},{"line_number":16,"context_line":"inlined-data segments."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This patch is a prerequisite for a future patch enabling the"},{"line_number":19,"context_line":"download of large objects as tarballs. The TLO patch will be added"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"3f4b6375_6d1be61a","line":16,"range":{"start_line":14,"start_character":14,"end_line":16,"end_character":22},"in_reply_to":"3f4b6375_eeba65dc","updated":"2017-11-01 12:09:40.000000000","message":"I seem to remember a discussion about the max number of segments being increased in a separate patch to 10000, but I\u0027m not confident of that memory any more! I really don\u0027t think that the data segment count needs to be checked - just an appropriate max_manifest_size.","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":"During a rolling upgrade, an updated proxy may write a manifest that"},{"line_number":24,"context_line":"out-of-date proxies will not be able to read. This will resolve itself"},{"line_number":25,"context_line":"once the upgrade completes on all nodes."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"Change-Id: Ib8dc216a84d370e6da7d6b819af79582b671d699"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":42,"id":"7f96bb07_e7deccd8","line":25,"updated":"2018-01-17 00:48:33.000000000","message":"I *think* this is a good idea?","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"00b23010d594a3887637098c47ea8bbe87c915e6","unresolved":false,"context_lines":[{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":"During a rolling upgrade, an updated proxy may write a manifest that"},{"line_number":24,"context_line":"out-of-date proxies will not be able to read. This will resolve itself"},{"line_number":25,"context_line":"once the upgrade completes on all nodes."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"Change-Id: Ib8dc216a84d370e6da7d6b819af79582b671d699"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":42,"id":"5f93b717_ad9bed63","line":25,"in_reply_to":"7f96bb07_e7deccd8","updated":"2018-01-30 19:38:33.000000000","message":"Seems reasonable. If clients aren\u0027t using the new data segments, then there\u0027s no impact on upgrade.","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"}],"swift/common/middleware/dlo.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3808ff07fc27e3653f795cca98d386e22d367c7b","unresolved":false,"context_lines":[{"line_number":191,"context_line":"                if isinstance(seg_name, six.text_type):"},{"line_number":192,"context_line":"                    seg_name \u003d seg_name.encode(\"utf-8\")"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"                # (obj path, etag, size, first byte, last byte)"},{"line_number":195,"context_line":"                yield (\"/\" + \"/\".join((version, account, container,"},{"line_number":196,"context_line":"                                       seg_name)),"},{"line_number":197,"context_line":"                       # We deliberately omit the etag and size here;"}],"source_content_type":"text/x-python","patch_set":26,"id":"9f436f4f_8b67974d","side":"PARENT","line":194,"updated":"2018-01-30 10:49:13.000000000","message":"This line used to explain the returning tuple, which made the comment below it make sense, omiting the etag and size. However this initial comment about the return structure is missing in therefore makes the comment out of context in the new patch.","commit_id":"b69b47c08cfe1234b0969984688c3ecbee50aae2"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":204,"context_line":"                    \u0027path\u0027: _path,"},{"line_number":205,"context_line":"                    \u0027first_byte\u0027: _first,"},{"line_number":206,"context_line":"                    \u0027last_byte\u0027: _last"},{"line_number":207,"context_line":"                }"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"                first_byte \u003d max(first_byte - seg_length, -1)"},{"line_number":210,"context_line":"                last_byte \u003d max(last_byte - seg_length, -1)"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_05c19be4","line":207,"updated":"2017-12-18 07:39:33.000000000","message":"ok, this changes the format from tuple to dict.\n\nPerhaps, it\u0027s better to add the info as the docs.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"}],"swift/common/middleware/slo.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d3e30614a5aff8fedc2fb12c948f476bacbc7198","unresolved":false,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"A DELETE with a query parameter::"},{"line_number":175,"context_line":"\"\"\""},{"line_number":176,"context_line":"\"\"\""},{"line_number":177,"context_line":"    ?multipart-manifest\u003ddelete"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"will delete all the segments referenced in the manifest and then the manifest"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_b1105d50","line":176,"updated":"2016-10-27 07:06:40.000000000","message":"?","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"29be545c47597e03ef38dc904518d7f7dd34a791","unresolved":false,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"A DELETE with a query parameter::"},{"line_number":175,"context_line":"\"\"\""},{"line_number":176,"context_line":"\"\"\""},{"line_number":177,"context_line":"    ?multipart-manifest\u003ddelete"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"will delete all the segments referenced in the manifest and then the manifest"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_0907151d","line":176,"in_reply_to":"ba5da102_b1105d50","updated":"2016-10-27 14:04:51.000000000","message":"Sorry, shitty syntax highlighter broken by the big string, I will remove :)","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d3e30614a5aff8fedc2fb12c948f476bacbc7198","unresolved":false,"context_lines":[{"line_number":237,"context_line":""},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"REQUIRED_SLO_KEYS \u003d set([\u0027path\u0027, \u0027etag\u0027, \u0027size_bytes\u0027])"},{"line_number":240,"context_line":"OPTIONAL_SLO_KEYS \u003d set([\u0027range\u0027, \u0027preamble\u0027, \u0027postamble\u0027])"},{"line_number":241,"context_line":"ALLOWED_SLO_KEYS \u003d REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_f11ad56d","line":240,"updated":"2016-10-27 07:06:40.000000000","message":"Oh man, this reminds me how I really want to move etag and size_bytes here, too.","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2bf12a82a82490f98e9c51a74d107cc766be0164","unresolved":false,"context_lines":[{"line_number":42,"context_line":"etag        (optional) the ETag given back when the segment object"},{"line_number":43,"context_line":"            was PUT"},{"line_number":44,"context_line":"size_bytes  (optional) the size of the complete segment object in"},{"line_number":45,"context_line":"            bytes"},{"line_number":46,"context_line":"range       (optional) the (inclusive) range within the object to"},{"line_number":47,"context_line":"            use as a segment. If omitted, the entire object is used."},{"line_number":48,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a6eadb0_7ba8e58d","line":45,"updated":"2016-12-16 14:06:17.000000000","message":"Add the new optional keys to the list","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":42,"context_line":"etag        (optional) the ETag given back when the segment object"},{"line_number":43,"context_line":"            was PUT"},{"line_number":44,"context_line":"size_bytes  (optional) the size of the complete segment object in"},{"line_number":45,"context_line":"            bytes"},{"line_number":46,"context_line":"range       (optional) the (inclusive) range within the object to"},{"line_number":47,"context_line":"            use as a segment. If omitted, the entire object is used."},{"line_number":48,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_dab06bf7","line":45,"in_reply_to":"1a6eadb0_7ba8e58d","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2bf12a82a82490f98e9c51a74d107cc766be0164","unresolved":false,"context_lines":[{"line_number":54,"context_line":"    [{\"path\": \"/cont/object\","},{"line_number":55,"context_line":"      \"etag\": \"etagoftheobjectsegment\","},{"line_number":56,"context_line":"      \"size_bytes\": 10485760,"},{"line_number":57,"context_line":"      \"range\": \"1048576-2097151\"}, ...]"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The number of object segments is limited to a configurable amount, default"},{"line_number":60,"context_line":"1000. Each segment must be at least 1 byte. On upload, the middleware will"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a6eadb0_bb9c3d5e","line":57,"updated":"2016-12-16 14:06:17.000000000","message":"add the new keys to the example","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":54,"context_line":"    [{\"path\": \"/cont/object\","},{"line_number":55,"context_line":"      \"etag\": \"etagoftheobjectsegment\","},{"line_number":56,"context_line":"      \"size_bytes\": 10485760,"},{"line_number":57,"context_line":"      \"range\": \"1048576-2097151\"}, ...]"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The number of object segments is limited to a configurable amount, default"},{"line_number":60,"context_line":"1000. Each segment must be at least 1 byte. On upload, the middleware will"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_fa952740","line":57,"in_reply_to":"1a6eadb0_bb9c3d5e","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2bf12a82a82490f98e9c51a74d107cc766be0164","unresolved":false,"context_lines":[{"line_number":65,"context_line":" 3. if the user provided a non-null etag, the etag matches;"},{"line_number":66,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"},{"line_number":67,"context_line":" 5. if the user provided a range, it is a singular, syntactically correct range"},{"line_number":68,"context_line":"    that is satisfiable given the size of the object."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Note that the etag and size_bytes keys are optional; if ommitted, the"},{"line_number":71,"context_line":"verification is not performed. If any of the objects fail to verify (not"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a6eadb0_1bac518b","line":68,"updated":"2016-12-16 14:06:17.000000000","message":"and yes, add the new keys here :-)","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":65,"context_line":" 3. if the user provided a non-null etag, the etag matches;"},{"line_number":66,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"},{"line_number":67,"context_line":" 5. if the user provided a range, it is a singular, syntactically correct range"},{"line_number":68,"context_line":"    that is satisfiable given the size of the object."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Note that the etag and size_bytes keys are optional; if ommitted, the"},{"line_number":71,"context_line":"verification is not performed. If any of the objects fail to verify (not"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_1a93d35c","line":68,"in_reply_to":"1a6eadb0_1bac518b","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":241,"context_line":"SYSMETA_SLO_SIZE \u003d get_sys_meta_prefix(\u0027object\u0027) + \u0027slo-size\u0027"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"def parse_and_validate_input(req_body, req_path):"},{"line_number":245,"context_line":"    \"\"\""},{"line_number":246,"context_line":"    Given a request body, parses it and returns a list of dictionaries."},{"line_number":247,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_77f7248b","line":244,"updated":"2016-12-27 20:28:58.000000000","message":"This should validate that any pre-/postambles decode properly, so we can fail invalid requests before making any HEADs. Also, the docstring will need updating.\n\nMaybe also drop any empty pre-/postambles here, rather than waiting until validate_seg_dict.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":241,"context_line":"SYSMETA_SLO_SIZE \u003d get_sys_meta_prefix(\u0027object\u0027) + \u0027slo-size\u0027"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"def parse_and_validate_input(req_body, req_path):"},{"line_number":245,"context_line":"    \"\"\""},{"line_number":246,"context_line":"    Given a request body, parses it and returns a list of dictionaries."},{"line_number":247,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_351ee063","line":244,"in_reply_to":"da4df55a_77f7248b","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":513,"context_line":"                # manipulation of first/last byte is done at the end of the"},{"line_number":514,"context_line":"                # loop when we\u0027re done with this segment."},{"line_number":515,"context_line":"                first_segment_byte \u003d range_start + max(0, first_byte)"},{"line_number":516,"context_line":"                last_segment_byte \u003d min(range_end, range_start + last_byte)"},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"                for sub_seg_dict in self._byterange_listing_iterator("},{"line_number":519,"context_line":"                        req, version, account, sub_segments,"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_cc83e354","line":516,"updated":"2016-12-27 20:28:58.000000000","message":"Should we hoist these up, out of the sub-SLO / normal object if block?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":513,"context_line":"                # manipulation of first/last byte is done at the end of the"},{"line_number":514,"context_line":"                # loop when we\u0027re done with this segment."},{"line_number":515,"context_line":"                first_segment_byte \u003d range_start + max(0, first_byte)"},{"line_number":516,"context_line":"                last_segment_byte \u003d min(range_end, range_start + last_byte)"},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"                for sub_seg_dict in self._byterange_listing_iterator("},{"line_number":519,"context_line":"                        req, version, account, sub_segments,"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_551b6c72","line":516,"in_reply_to":"da4df55a_cc83e354","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":518,"context_line":"                for sub_seg_dict in self._byterange_listing_iterator("},{"line_number":519,"context_line":"                        req, version, account, sub_segments,"},{"line_number":520,"context_line":"                        # This adjusts first_byte and last_byte to be"},{"line_number":521,"context_line":"                        # relative to the sub-SLO."},{"line_number":522,"context_line":"                        first_segment_byte, last_segment_byte,"},{"line_number":523,"context_line":"                        cached_fetch_sub_slo_segments,"},{"line_number":524,"context_line":"                        recursion_depth\u003drecursion_depth + 1):"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_ac5dbf16","line":521,"updated":"2016-12-27 20:28:58.000000000","message":"Probably OK to drop this comment since we have the longer explanation above.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":518,"context_line":"                for sub_seg_dict in self._byterange_listing_iterator("},{"line_number":519,"context_line":"                        req, version, account, sub_segments,"},{"line_number":520,"context_line":"                        # This adjusts first_byte and last_byte to be"},{"line_number":521,"context_line":"                        # relative to the sub-SLO."},{"line_number":522,"context_line":"                        first_segment_byte, last_segment_byte,"},{"line_number":523,"context_line":"                        cached_fetch_sub_slo_segments,"},{"line_number":524,"context_line":"                        recursion_depth\u003drecursion_depth + 1):"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_95795496","line":521,"in_reply_to":"da4df55a_ac5dbf16","updated":"2017-01-13 23:35:22.000000000","message":"tbh, it took me longer than I\u0027d like to admit to fathom out what was going on here, so I\u0027d rather leave both comments in if you don\u0027t object.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":531,"context_line":"                path \u003d \"/{ver}/{acc}/{conobj}\".format("},{"line_number":532,"context_line":"                    ver\u003dversion, acc\u003daccount,"},{"line_number":533,"context_line":"                    conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":534,"context_line":"                )"},{"line_number":535,"context_line":"                seg_dict[\u0027path\u0027] \u003d path"},{"line_number":536,"context_line":"                seg_dict[\u0027first_byte\u0027] \u003d first_segment_byte"},{"line_number":537,"context_line":"                seg_dict[\u0027last_byte\u0027] \u003d last_segment_byte"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_ace21f2d","line":534,"updated":"2016-12-27 20:28:58.000000000","message":"Do we have a preferred way to do path construction? Over in DLO, we\u0027ve got something like\n\n \"/\".join([...])\n\nI think both are readable, so maybe it just doesn\u0027t matter?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":751,"context_line":""},{"line_number":752,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":753,"context_line":"            if seg_postamble:"},{"line_number":754,"context_line":"                seg_dict[\u0027postamble\u0027] \u003d base64.b64decode(seg_postamble)"},{"line_number":755,"context_line":""},{"line_number":756,"context_line":"            if slo_etag is None:"},{"line_number":757,"context_line":"                if seg_preamble:"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_ec950733","line":754,"updated":"2016-12-27 20:28:58.000000000","message":"Maybe deserves a comment that the pre-/postamble decoding must happen *before* the _segment_length calculation?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":751,"context_line":""},{"line_number":752,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":753,"context_line":"            if seg_postamble:"},{"line_number":754,"context_line":"                seg_dict[\u0027postamble\u0027] \u003d base64.b64decode(seg_postamble)"},{"line_number":755,"context_line":""},{"line_number":756,"context_line":"            if slo_etag is None:"},{"line_number":757,"context_line":"                if seg_preamble:"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_d5f83ce8","line":754,"in_reply_to":"da4df55a_ec950733","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":757,"context_line":"                if seg_preamble:"},{"line_number":758,"context_line":"                    _pre_md5 \u003d md5()"},{"line_number":759,"context_line":"                    _pre_md5.update(seg_dict[\u0027preamble\u0027])"},{"line_number":760,"context_line":"                    calculated_etag.update(_pre_md5.hexdigest())"},{"line_number":761,"context_line":"                if seg_dict.get(\u0027range\u0027):"},{"line_number":762,"context_line":"                    calculated_etag.update("},{"line_number":763,"context_line":"                        \u0027%s:%s;\u0027 % (seg_dict[\u0027hash\u0027], seg_dict[\u0027range\u0027])"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_4cfa9376","line":760,"updated":"2016-12-27 20:28:58.000000000","message":"Interesting. So if you move the preamble to a separate object and replace the manifest with one that references the new object instead of having a preamble, both manifests have the same ETag? I rather like that.\n\nI\u0027m not sure I like the extra MD5-ing on every download, though; is it necessary? Wouldn\u0027t the presence of a pre-/postamble imply that the SLO was created after Ia6ad32354105515560b005cea750aa64a88c96f9 landed?\n\nOr is this \"just in case\", like our call to override_bytes_from_content_type below? (Might be worth looking at the comment around https://github.com/openstack/swift/blob/2.12.0/test/unit/common/middleware/test_slo.py#L1390 -- I\u0027m not convinced that situation should ever arise.)","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":757,"context_line":"                if seg_preamble:"},{"line_number":758,"context_line":"                    _pre_md5 \u003d md5()"},{"line_number":759,"context_line":"                    _pre_md5.update(seg_dict[\u0027preamble\u0027])"},{"line_number":760,"context_line":"                    calculated_etag.update(_pre_md5.hexdigest())"},{"line_number":761,"context_line":"                if seg_dict.get(\u0027range\u0027):"},{"line_number":762,"context_line":"                    calculated_etag.update("},{"line_number":763,"context_line":"                        \u0027%s:%s;\u0027 % (seg_dict[\u0027hash\u0027], seg_dict[\u0027range\u0027])"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_0738d7d2","line":760,"in_reply_to":"da4df55a_4cfa9376","updated":"2017-01-13 23:35:22.000000000","message":"You make a convincing argument that we shouldn\u0027t need to calculate the etag... but given that the test for an existing  etag is so cheap, I kind of think there\u0027s no harm in leaving it for now. I\u0027d really need to convince myself completely that it\u0027s not needed before removing this code... maybe a follow-up patch?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":809,"context_line":"            req, ver, account, segments, byteranges"},{"line_number":810,"context_line":"        )"},{"line_number":811,"context_line":""},{"line_number":812,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":813,"context_line":"            start_byte \u003d seg_dict.get(\u0027first_byte\u0027)"},{"line_number":814,"context_line":"            start \u003d 0 if start_byte is None else start_byte"},{"line_number":815,"context_line":"            end_byte \u003d seg_dict.get(\u0027end_byte\u0027)"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_0c396bf2","line":812,"updated":"2016-12-27 20:28:58.000000000","message":"Ah, that feels better :-)","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":811,"context_line":""},{"line_number":812,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":813,"context_line":"            start_byte \u003d seg_dict.get(\u0027first_byte\u0027)"},{"line_number":814,"context_line":"            start \u003d 0 if start_byte is None else start_byte"},{"line_number":815,"context_line":"            end_byte \u003d seg_dict.get(\u0027end_byte\u0027)"},{"line_number":816,"context_line":"            end \u003d int(seg_dict[\u0027bytes\u0027]) - 1 if end_byte is None else end_byte"},{"line_number":817,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_7712c458","line":814,"updated":"2016-12-27 20:28:58.000000000","message":"Maybe just\n\n start \u003d seg_dict.get(\u0027first_byte\u0027) or 0\n\n? Or even\n\n start \u003d seg_dict.get(\u0027first_byte\u0027, 0)\n\nWould seg_dict[\u0027first_byte\u0027] ever be set, but to None?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":811,"context_line":""},{"line_number":812,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":813,"context_line":"            start_byte \u003d seg_dict.get(\u0027first_byte\u0027)"},{"line_number":814,"context_line":"            start \u003d 0 if start_byte is None else start_byte"},{"line_number":815,"context_line":"            end_byte \u003d seg_dict.get(\u0027end_byte\u0027)"},{"line_number":816,"context_line":"            end \u003d int(seg_dict[\u0027bytes\u0027]) - 1 if end_byte is None else end_byte"},{"line_number":817,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_c7f88fe8","line":814,"in_reply_to":"da4df55a_7712c458","updated":"2017-01-13 23:35:22.000000000","message":"It shouldn\u0027t, so let\u0027s shortcut this.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":813,"context_line":"            start_byte \u003d seg_dict.get(\u0027first_byte\u0027)"},{"line_number":814,"context_line":"            start \u003d 0 if start_byte is None else start_byte"},{"line_number":815,"context_line":"            end_byte \u003d seg_dict.get(\u0027end_byte\u0027)"},{"line_number":816,"context_line":"            end \u003d int(seg_dict[\u0027bytes\u0027]) - 1 if end_byte is None else end_byte"},{"line_number":817,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"},{"line_number":818,"context_line":"            return is_small"},{"line_number":819,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_976008d1","line":816,"updated":"2016-12-27 20:28:58.000000000","message":"If seg_dict[\u0027end_byte\u0027] would never be set to None, this could be reduced to\n\n end \u003d seg_dict.get(\u0027end_byte\u0027, int(seg_dict[\u0027bytes\u0027]) - 1)","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":813,"context_line":"            start_byte \u003d seg_dict.get(\u0027first_byte\u0027)"},{"line_number":814,"context_line":"            start \u003d 0 if start_byte is None else start_byte"},{"line_number":815,"context_line":"            end_byte \u003d seg_dict.get(\u0027end_byte\u0027)"},{"line_number":816,"context_line":"            end \u003d int(seg_dict[\u0027bytes\u0027]) - 1 if end_byte is None else end_byte"},{"line_number":817,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"},{"line_number":818,"context_line":"            return is_small"},{"line_number":819,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_e7dd8b71","line":816,"in_reply_to":"da4df55a_976008d1","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":825,"context_line":""},{"line_number":826,"context_line":"        def segment_path_iter():"},{"line_number":827,"context_line":"            for seg_dict in ratelimited_listing_iter:"},{"line_number":828,"context_line":"                seg_dict[\u0027path\u0027] \u003d \"/{ver}/{acc}/{conobj}\".format("},{"line_number":829,"context_line":"                    ver\u003dver, acc\u003daccount, conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":830,"context_line":"                )"},{"line_number":831,"context_line":"                yield seg_dict"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_b7ec2cf4","line":828,"updated":"2016-12-27 20:28:58.000000000","message":"Now that we loop over all segments in get_or_head_response anyway, should we set seg_dict[\u0027path\u0027] there as well and nix this whole generator?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":825,"context_line":""},{"line_number":826,"context_line":"        def segment_path_iter():"},{"line_number":827,"context_line":"            for seg_dict in ratelimited_listing_iter:"},{"line_number":828,"context_line":"                seg_dict[\u0027path\u0027] \u003d \"/{ver}/{acc}/{conobj}\".format("},{"line_number":829,"context_line":"                    ver\u003dver, acc\u003daccount, conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":830,"context_line":"                )"},{"line_number":831,"context_line":"                yield seg_dict"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_fd6fde8f","line":828,"in_reply_to":"da4df55a_b7ec2cf4","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2bf12a82a82490f98e9c51a74d107cc766be0164","unresolved":false,"context_lines":[{"line_number":828,"context_line":"                seg_dict[\u0027path\u0027] \u003d \"/{ver}/{acc}/{conobj}\".format("},{"line_number":829,"context_line":"                    ver\u003dver, acc\u003daccount, conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":830,"context_line":"                )"},{"line_number":831,"context_line":"                yield seg_dict"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"        segmented_iter \u003d SegmentedIterable("},{"line_number":834,"context_line":"            req, self.slo.app, segment_path_iter(),"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a6eadb0_5beb2913","line":831,"updated":"2016-12-16 14:06:17.000000000","message":"This path determination is also done at line 531.  It might be nice to have that captured in a function or something for code reuse.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":828,"context_line":"                seg_dict[\u0027path\u0027] \u003d \"/{ver}/{acc}/{conobj}\".format("},{"line_number":829,"context_line":"                    ver\u003dver, acc\u003daccount, conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":830,"context_line":"                )"},{"line_number":831,"context_line":"                yield seg_dict"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"        segmented_iter \u003d SegmentedIterable("},{"line_number":834,"context_line":"            req, self.slo.app, segment_path_iter(),"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_3d6a4681","line":831,"in_reply_to":"1a6eadb0_5beb2913","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":986,"context_line":"            if segment_length \u003c 1:"},{"line_number":987,"context_line":"                problem_segments.append("},{"line_number":988,"context_line":"                    [quote(obj_name),"},{"line_number":989,"context_line":"                     \u0027Too small; each segment must be at least 1 byte.\u0027])"},{"line_number":990,"context_line":""},{"line_number":991,"context_line":"            _size_bytes \u003d seg_dict.get(\u0027size_bytes\u0027)"},{"line_number":992,"context_line":"            size_mismatch \u003d ("}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_770ba42a","line":989,"updated":"2016-12-27 20:28:58.000000000","message":"Should this still be an error if there was a pre-postamble? I\u0027m not actually sure; just want to bring it up for discussion.\n\nWill the future TLO stuff allow zero-byte files in the archive?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":986,"context_line":"            if segment_length \u003c 1:"},{"line_number":987,"context_line":"                problem_segments.append("},{"line_number":988,"context_line":"                    [quote(obj_name),"},{"line_number":989,"context_line":"                     \u0027Too small; each segment must be at least 1 byte.\u0027])"},{"line_number":990,"context_line":""},{"line_number":991,"context_line":"            _size_bytes \u003d seg_dict.get(\u0027size_bytes\u0027)"},{"line_number":992,"context_line":"            size_mismatch \u003d ("}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_5dd39238","line":989,"in_reply_to":"da4df55a_770ba42a","updated":"2017-01-13 23:35:22.000000000","message":"Yeah, you\u0027re right, we do want 0 byte objects with pre/postambles and this is a good place to sort it :)","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":1030,"context_line":"                segment_length +\u003d len(base64.b64decode(seg_dict[\u0027preamble\u0027]))"},{"line_number":1031,"context_line":"            if seg_dict.get(\u0027postamble\u0027):"},{"line_number":1032,"context_line":"                seg_data[\u0027postamble\u0027] \u003d seg_dict[\u0027postamble\u0027]"},{"line_number":1033,"context_line":"                segment_length +\u003d len(base64.b64decode(seg_dict[\u0027postamble\u0027]))"},{"line_number":1034,"context_line":"            return segment_length, seg_data"},{"line_number":1035,"context_line":""},{"line_number":1036,"context_line":"        data_for_storage \u003d [None] * len(parsed_data)"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_d7645023","line":1033,"updated":"2016-12-27 20:28:58.000000000","message":"Maybe worth a short\n\n for key in (\u0027preamble\u0027, \u0027postamble\u0027):\n\nloop? I could go either way.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":1030,"context_line":"                segment_length +\u003d len(base64.b64decode(seg_dict[\u0027preamble\u0027]))"},{"line_number":1031,"context_line":"            if seg_dict.get(\u0027postamble\u0027):"},{"line_number":1032,"context_line":"                seg_data[\u0027postamble\u0027] \u003d seg_dict[\u0027postamble\u0027]"},{"line_number":1033,"context_line":"                segment_length +\u003d len(base64.b64decode(seg_dict[\u0027postamble\u0027]))"},{"line_number":1034,"context_line":"            return segment_length, seg_data"},{"line_number":1035,"context_line":""},{"line_number":1036,"context_line":"        data_for_storage \u003d [None] * len(parsed_data)"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_7d7e8ee5","line":1033,"in_reply_to":"da4df55a_d7645023","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    [{\"path\": \"/cont/object\","},{"line_number":59,"context_line":"      \"etag\": \"etagoftheobjectsegment\","},{"line_number":60,"context_line":"      \"size_bytes\": 10485760,"},{"line_number":61,"context_line":"      \"range\": \"1048576-2097151\"},"},{"line_number":62,"context_line":"      \"preamble\": base64.b64encode(\"preamble\"),"},{"line_number":63,"context_line":"      \"postamble\": base64.b64encode(\"postamble\") ...]"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_2635cff6","line":61,"range":{"start_line":61,"start_character":32,"end_line":61,"end_character":34},"updated":"2017-03-01 22:48:25.000000000","message":"Not valid pseudo-json any more...","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    [{\"path\": \"/cont/object\","},{"line_number":59,"context_line":"      \"etag\": \"etagoftheobjectsegment\","},{"line_number":60,"context_line":"      \"size_bytes\": 10485760,"},{"line_number":61,"context_line":"      \"range\": \"1048576-2097151\"},"},{"line_number":62,"context_line":"      \"preamble\": base64.b64encode(\"preamble\"),"},{"line_number":63,"context_line":"      \"postamble\": base64.b64encode(\"postamble\") ...]"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_6da718bd","line":61,"range":{"start_line":61,"start_character":32,"end_line":61,"end_character":34},"in_reply_to":"ba2be162_2635cff6","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    [{\"path\": \"/cont/object\","},{"line_number":59,"context_line":"      \"etag\": \"etagoftheobjectsegment\","},{"line_number":60,"context_line":"      \"size_bytes\": 10485760,"},{"line_number":61,"context_line":"      \"range\": \"1048576-2097151\"},"},{"line_number":62,"context_line":"      \"preamble\": base64.b64encode(\"preamble\"),"},{"line_number":63,"context_line":"      \"postamble\": base64.b64encode(\"postamble\") ...]"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_9b16bba5","line":61,"range":{"start_line":61,"start_character":32,"end_line":61,"end_character":34},"in_reply_to":"ba2be162_2635cff6","updated":"2017-03-02 11:41:56.000000000","message":"Will fix.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":68,"context_line":"verify:"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":" 1. the segment exists (i.e. the HEAD was successful);"},{"line_number":71,"context_line":" 2. the segment combined with preamble and postamble data meets minimum size"},{"line_number":72,"context_line":"    requirements;"},{"line_number":73,"context_line":" 3. if the user provided a non-null etag, the etag matches;"},{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_860d9b40","line":71,"range":{"start_line":71,"start_character":16,"end_line":71,"end_character":29},"updated":"2017-03-01 22:48:25.000000000","message":"nit: \"including\", maybe?","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":68,"context_line":"verify:"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":" 1. the segment exists (i.e. the HEAD was successful);"},{"line_number":71,"context_line":" 2. the segment combined with preamble and postamble data meets minimum size"},{"line_number":72,"context_line":"    requirements;"},{"line_number":73,"context_line":" 3. if the user provided a non-null etag, the etag matches;"},{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_8daa8c83","line":71,"range":{"start_line":71,"start_character":16,"end_line":71,"end_character":29},"in_reply_to":"ba2be162_4c7ac8f2","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":68,"context_line":"verify:"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":" 1. the segment exists (i.e. the HEAD was successful);"},{"line_number":71,"context_line":" 2. the segment combined with preamble and postamble data meets minimum size"},{"line_number":72,"context_line":"    requirements;"},{"line_number":73,"context_line":" 3. if the user provided a non-null etag, the etag matches;"},{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_4c7ac8f2","line":71,"range":{"start_line":71,"start_character":16,"end_line":71,"end_character":29},"in_reply_to":"ba2be162_860d9b40","updated":"2017-03-02 11:41:56.000000000","message":"yes, I think that\u0027s better.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"},{"line_number":75,"context_line":" 5. if the user provided a range, it is a singular, syntactically correct range"},{"line_number":76,"context_line":"    that is satisfiable given the size of the object and any preamble and/or"},{"line_number":77,"context_line":"    postamble data."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"Note that the etag and size_bytes keys are optional; if ommitted, the"},{"line_number":80,"context_line":"verification is not performed. If any of the objects fail to verify (not"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_e119e17a","line":77,"updated":"2017-03-01 22:48:25.000000000","message":"Wait, the range slices into the *ambles? That doesn\u0027t seem to match what the code is doing :-/\n\nI think Janie meant we should add something like\n\n\u003e 6. if the user provided a preamble or postamble, it is valid base64 data.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"},{"line_number":75,"context_line":" 5. if the user provided a range, it is a singular, syntactically correct range"},{"line_number":76,"context_line":"    that is satisfiable given the size of the object and any preamble and/or"},{"line_number":77,"context_line":"    postamble data."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"Note that the etag and size_bytes keys are optional; if ommitted, the"},{"line_number":80,"context_line":"verification is not performed. If any of the objects fail to verify (not"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_adad909c","line":77,"in_reply_to":"ba2be162_db391343","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":74,"context_line":" 4. if the user provided a non-null size_bytes, the size_bytes matches; and"},{"line_number":75,"context_line":" 5. if the user provided a range, it is a singular, syntactically correct range"},{"line_number":76,"context_line":"    that is satisfiable given the size of the object and any preamble and/or"},{"line_number":77,"context_line":"    postamble data."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"Note that the etag and size_bytes keys are optional; if ommitted, the"},{"line_number":80,"context_line":"verification is not performed. If any of the objects fail to verify (not"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_db391343","line":77,"in_reply_to":"ba2be162_e119e17a","updated":"2017-03-02 11:41:56.000000000","message":"No, no it doesn\u0027t. I just put this doc update in the wrong place. Sorry for the confusion.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        # Validate that preamble is valid base64 encoded data or remove"},{"line_number":422,"context_line":"        # empty preambles"},{"line_number":423,"context_line":"        if \u0027preamble\u0027 in seg_dict:"},{"line_number":424,"context_line":"            if seg_dict.get(\u0027preamble\u0027):"},{"line_number":425,"context_line":"                try:"},{"line_number":426,"context_line":"                    base64.b64decode(seg_dict[\u0027preamble\u0027])"},{"line_number":427,"context_line":"                except TypeError:"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_c14cdd72","line":424,"updated":"2017-03-01 22:48:25.000000000","message":"nit: We already know it\u0027s in there; may as well say\n\n seg_dict[\u0027preamble\u0027]","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        # Validate that preamble is valid base64 encoded data or remove"},{"line_number":422,"context_line":"        # empty preambles"},{"line_number":423,"context_line":"        if \u0027preamble\u0027 in seg_dict:"},{"line_number":424,"context_line":"            if seg_dict.get(\u0027preamble\u0027):"},{"line_number":425,"context_line":"                try:"},{"line_number":426,"context_line":"                    base64.b64decode(seg_dict[\u0027preamble\u0027])"},{"line_number":427,"context_line":"                except TypeError:"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_fb033740","line":424,"in_reply_to":"ba2be162_c14cdd72","updated":"2017-03-02 11:41:56.000000000","message":"Good point :)","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        # Validate that preamble is valid base64 encoded data or remove"},{"line_number":422,"context_line":"        # empty preambles"},{"line_number":423,"context_line":"        if \u0027preamble\u0027 in seg_dict:"},{"line_number":424,"context_line":"            if seg_dict.get(\u0027preamble\u0027):"},{"line_number":425,"context_line":"                try:"},{"line_number":426,"context_line":"                    base64.b64decode(seg_dict[\u0027preamble\u0027])"},{"line_number":427,"context_line":"                except TypeError:"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_cd98047b","line":424,"in_reply_to":"ba2be162_fb033740","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":431,"context_line":"                    )"},{"line_number":432,"context_line":"                    continue"},{"line_number":433,"context_line":"            else:"},{"line_number":434,"context_line":"                del seg_dict[\u0027preamble\u0027]"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"        # Validate that postamble is valid base64 encoded data or remove"},{"line_number":437,"context_line":"        # empty postambles"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_816905c1","line":434,"updated":"2017-03-01 22:48:25.000000000","message":"So we\u0027ll drop preambles from dicts like\n\n {\"preamble\": null, ...}\n {\"preamble\": \"\", ...}\n {\"preamble\": 0, ...}\n {\"preamble\": false, ...}\n\nThe first two I can get behind, but the last two seem strange, particularly given how 1 or true would cause errors.\n\nAlso, I think the preamble from\n\n {\"preamble\": \"\u003d\u003d\u003d\", ...}\n\nstill gets written to disk? Which seems strange/pointless.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":431,"context_line":"                    )"},{"line_number":432,"context_line":"                    continue"},{"line_number":433,"context_line":"            else:"},{"line_number":434,"context_line":"                del seg_dict[\u0027preamble\u0027]"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"        # Validate that postamble is valid base64 encoded data or remove"},{"line_number":437,"context_line":"        # empty postambles"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_ed930859","line":434,"in_reply_to":"ba2be162_4cd4881f","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":431,"context_line":"                    )"},{"line_number":432,"context_line":"                    continue"},{"line_number":433,"context_line":"            else:"},{"line_number":434,"context_line":"                del seg_dict[\u0027preamble\u0027]"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"        # Validate that postamble is valid base64 encoded data or remove"},{"line_number":437,"context_line":"        # empty postambles"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_4cd4881f","line":434,"in_reply_to":"ba2be162_816905c1","updated":"2017-03-02 11:41:56.000000000","message":"Yeah, you make some valid points here, but you\u0027ve highlighted to me a couple of other things that need work.\n\nI\u0027m absolutely not worried about deleting 0/False because the 1/True case wouldn\u0027t pass validation anyway (the TypeError). The issue is that a failed b64decode on py3 raises a binascii.Error not a TypeError, so thanks for making me look harder at this :)\n\nI believe we should be deleting the key if b64decode returns b\u0027\u0027 like you say, so I\u0027ll rejig things and add the extra error handling.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":838,"context_line":"        if content_length is None:"},{"line_number":839,"context_line":"            calculated_content_length \u003d 0"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        if slo_etag is None or content_length is None:"},{"line_number":842,"context_line":"            for seg_dict in segments:"},{"line_number":843,"context_line":"                # It\u0027s important that we decode the preamble and postamble"},{"line_number":844,"context_line":"                # here before we calculate the segment length (_segment_length)"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_21d4d945","line":841,"updated":"2017-03-01 22:48:25.000000000","message":"I don\u0027t think we want this. Don\u0027t we have to do the decoding even if we got slo_etag/content_length from sysmeta?\n\nDo we have func tests?","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":838,"context_line":"        if content_length is None:"},{"line_number":839,"context_line":"            calculated_content_length \u003d 0"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        if slo_etag is None or content_length is None:"},{"line_number":842,"context_line":"            for seg_dict in segments:"},{"line_number":843,"context_line":"                # It\u0027s important that we decode the preamble and postamble"},{"line_number":844,"context_line":"                # here before we calculate the segment length (_segment_length)"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_6c930c50","line":841,"in_reply_to":"ba2be162_21d4d945","updated":"2017-03-02 11:41:56.000000000","message":"I\u0027ll ping you to chat about this.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":838,"context_line":"        if content_length is None:"},{"line_number":839,"context_line":"            calculated_content_length \u003d 0"},{"line_number":840,"context_line":""},{"line_number":841,"context_line":"        if slo_etag is None or content_length is None:"},{"line_number":842,"context_line":"            for seg_dict in segments:"},{"line_number":843,"context_line":"                # It\u0027s important that we decode the preamble and postamble"},{"line_number":844,"context_line":"                # here before we calculate the segment length (_segment_length)"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_2d9a806e","line":841,"in_reply_to":"ba2be162_6c930c50","updated":"2017-03-05 11:54:07.000000000","message":"Okay, I\u0027ve fixed everything else, will ping you on Monday about this.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":1072,"context_line":"                    seg_dict[\u0027range\u0027] \u003d \u0027%d-%d\u0027 % (rng[0], rng[1] - 1)"},{"line_number":1073,"context_line":"                    segment_length \u003d rng[1] - rng[0]"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            seg_extra_data \u003d {}"},{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_216219ae","line":1075,"updated":"2017-03-01 22:48:25.000000000","message":"Could just call it seg_data now, and do a .update({...}) later.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":1072,"context_line":"                    seg_dict[\u0027range\u0027] \u003d \u0027%d-%d\u0027 % (rng[0], rng[1] - 1)"},{"line_number":1073,"context_line":"                    segment_length \u003d rng[1] - rng[0]"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            seg_extra_data \u003d {}"},{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_ccd23800","line":1075,"in_reply_to":"ba2be162_216219ae","updated":"2017-03-02 11:41:56.000000000","message":"I\u0027ll have a look at that.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":1072,"context_line":"                    seg_dict[\u0027range\u0027] \u003d \u0027%d-%d\u0027 % (rng[0], rng[1] - 1)"},{"line_number":1073,"context_line":"                    segment_length \u003d rng[1] - rng[0]"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            seg_extra_data \u003d {}"},{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_4d8b341f","line":1075,"in_reply_to":"ba2be162_ccd23800","updated":"2017-03-05 11:54:07.000000000","message":"Done","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"},{"line_number":1079,"context_line":"                    segment_length +\u003d len(base64.b64decode(seg_dict[key]))"},{"line_number":1080,"context_line":"            if segment_length \u003c 1:"},{"line_number":1081,"context_line":"                problem_segments.append("},{"line_number":1082,"context_line":"                    [quote(obj_name),"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_c1c93d7a","line":1079,"updated":"2017-03-01 22:48:25.000000000","message":"I maybe would\u0027ve preferred to not touch segment_length but instead have\n\n if segment_length \u003c 1 and \u0027preamble\u0027 not in seg_dict and \u0027postamble\u0027 not in seg_dict:\n\nbelow... but I guess we never use segment_length again? Meh. As long as we\u0027re writing head_seg_resp.content_length down to disk...","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"},{"line_number":1079,"context_line":"                    segment_length +\u003d len(base64.b64decode(seg_dict[key]))"},{"line_number":1080,"context_line":"            if segment_length \u003c 1:"},{"line_number":1081,"context_line":"                problem_segments.append("},{"line_number":1082,"context_line":"                    [quote(obj_name),"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_4da414b9","line":1079,"in_reply_to":"ba2be162_8c0da092","updated":"2017-03-05 11:54:07.000000000","message":"Having looked at this, I think that modifying segment_length is the right thing to do, because for the purposes of this code, the segment_length really does include pre/postamble data.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"0939e8ccda1492b6ef67e9c075e44f467c104ab4","unresolved":false,"context_lines":[{"line_number":1076,"context_line":"            for key in [\u0027preamble\u0027, \u0027postamble\u0027]:"},{"line_number":1077,"context_line":"                if seg_dict.get(key):"},{"line_number":1078,"context_line":"                    seg_extra_data[key] \u003d seg_dict[key]"},{"line_number":1079,"context_line":"                    segment_length +\u003d len(base64.b64decode(seg_dict[key]))"},{"line_number":1080,"context_line":"            if segment_length \u003c 1:"},{"line_number":1081,"context_line":"                problem_segments.append("},{"line_number":1082,"context_line":"                    [quote(obj_name),"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_8c0da092","line":1079,"in_reply_to":"ba2be162_c1c93d7a","updated":"2017-03-02 11:41:56.000000000","message":"Yeah, I suppose... I was kind of thinking of the slo-segment as being the object we\u0027re including along with any pre/post amble data, rather than considering them as separate (a segment dict is seg-data+pre+post...)","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fb3972fdde6379cfd8a906db5d7d6c7451b53599","unresolved":false,"context_lines":[{"line_number":191,"context_line":"  .. note::"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"     The this feature is primarily aimed at reducing the need for storing"},{"line_number":195,"context_line":"     many tiny objects, and as such any supplied pre/post-amble data must"},{"line_number":196,"context_line":"     fit within the maximum manifest size (default is 8MiB). This maximum"},{"line_number":197,"context_line":"     size can be configured by setting \u0027max_manifest_size\u0027."}],"source_content_type":"text/x-python","patch_set":21,"id":"7ffa3b31_c72cbf03","line":194,"range":{"start_line":194,"start_character":5,"end_line":194,"end_character":13},"updated":"2017-04-21 01:41:09.000000000","message":"s/The this/This/","commit_id":"ebc637c450b56cccbd8fb010b4c1498bae88ddef"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"6ec89f668433a158dbfbeb3e562730457183b526","unresolved":false,"context_lines":[{"line_number":191,"context_line":"  .. note::"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"     The this feature is primarily aimed at reducing the need for storing"},{"line_number":195,"context_line":"     many tiny objects, and as such any supplied pre/post-amble data must"},{"line_number":196,"context_line":"     fit within the maximum manifest size (default is 8MiB). This maximum"},{"line_number":197,"context_line":"     size can be configured by setting \u0027max_manifest_size\u0027."}],"source_content_type":"text/x-python","patch_set":21,"id":"7ffa3b31_fc361e51","line":194,"range":{"start_line":194,"start_character":5,"end_line":194,"end_character":13},"in_reply_to":"7ffa3b31_c72cbf03","updated":"2017-04-21 16:12:48.000000000","message":"Good spot :)","commit_id":"ebc637c450b56cccbd8fb010b4c1498bae88ddef"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fb3972fdde6379cfd8a906db5d7d6c7451b53599","unresolved":false,"context_lines":[{"line_number":852,"context_line":"                # here before we calculate the segment length (_segment_length)"},{"line_number":853,"context_line":"                seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":854,"context_line":"                if seg_preamble:"},{"line_number":855,"context_line":"                    seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":856,"context_line":""},{"line_number":857,"context_line":"                seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":858,"context_line":"                if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":21,"id":"7ffa3b31_4afaf6bc","line":855,"updated":"2017-04-21 01:41:09.000000000","message":"Yeah, this decoding still needs to come out of the if -- otherwise, I get the base64 version on download.\n\nI\u0027ll see about writing some func tests tomorrow to help demonstrate the problem.","commit_id":"ebc637c450b56cccbd8fb010b4c1498bae88ddef"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"6ec89f668433a158dbfbeb3e562730457183b526","unresolved":false,"context_lines":[{"line_number":852,"context_line":"                # here before we calculate the segment length (_segment_length)"},{"line_number":853,"context_line":"                seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":854,"context_line":"                if seg_preamble:"},{"line_number":855,"context_line":"                    seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":856,"context_line":""},{"line_number":857,"context_line":"                seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":858,"context_line":"                if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":21,"id":"7ffa3b31_fce85eb3","line":855,"in_reply_to":"7ffa3b31_4afaf6bc","updated":"2017-04-21 16:12:48.000000000","message":"I\u0027ll have a play with this too.","commit_id":"ebc637c450b56cccbd8fb010b4c1498bae88ddef"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"779faedafa11b86424b041f4c63baf2a6ae21801","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"DEFAULT_RATE_LIMIT_UNDER_SIZE \u003d 1024 ** 2  # 1 MiB"},{"line_number":318,"context_line":"DEFAULT_MAX_MANIFEST_SEGMENTS \u003d 1000"},{"line_number":319,"context_line":"DEFAULT_MAX_MANIFEST_SIZE \u003d 8 * (1024 ** 2)  # 8 MiB"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"REQUIRED_SLO_KEYS \u003d set([\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":24,"id":"7f231b9d_f9846860","line":319,"updated":"2017-06-13 07:47:36.000000000","message":"So we increase the default max manifest size from 2 to 8 MiB - I think this needs to be stated in the commit message as well? Some operators might be surprised by this?","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"5b6edea2751ea64b61c9f824420610a748cc1ddb","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"DEFAULT_RATE_LIMIT_UNDER_SIZE \u003d 1024 ** 2  # 1 MiB"},{"line_number":318,"context_line":"DEFAULT_MAX_MANIFEST_SEGMENTS \u003d 1000"},{"line_number":319,"context_line":"DEFAULT_MAX_MANIFEST_SIZE \u003d 8 * (1024 ** 2)  # 8 MiB"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"REQUIRED_SLO_KEYS \u003d set([\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_274a8eab","line":319,"in_reply_to":"7f231b9d_f9846860","updated":"2017-08-02 19:14:53.000000000","message":"Agreed","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"ffdedb57d90f296625a8100f60006a0b3749aa80","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"DEFAULT_RATE_LIMIT_UNDER_SIZE \u003d 1024 ** 2  # 1 MiB"},{"line_number":318,"context_line":"DEFAULT_MAX_MANIFEST_SEGMENTS \u003d 1000"},{"line_number":319,"context_line":"DEFAULT_MAX_MANIFEST_SIZE \u003d 8 * (1024 ** 2)  # 8 MiB"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"REQUIRED_SLO_KEYS \u003d set([\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_ea0b2bd3","line":319,"in_reply_to":"7f231b9d_f9846860","updated":"2017-08-02 19:32:33.000000000","message":"Done","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"779faedafa11b86424b041f4c63baf2a6ae21801","unresolved":false,"context_lines":[{"line_number":463,"context_line":"                        del seg_dict[key]"},{"line_number":464,"context_line":"        except (TypeError, binascii.Error):"},{"line_number":465,"context_line":"            errors.append("},{"line_number":466,"context_line":"                \"Index %d: %s must be valid base64 encoded data\""},{"line_number":467,"context_line":"                % (seg_index, key)"},{"line_number":468,"context_line":"            )"},{"line_number":469,"context_line":"            continue"}],"source_content_type":"text/x-python","patch_set":24,"id":"7f231b9d_b90f10e0","line":466,"updated":"2017-06-13 07:47:36.000000000","message":"Hmm, despite sending a valid JSON this results in a 400/Bad Request with \"Manifest must be valid JSON.\" error. That might be not as helpful as it should be?","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"5b6edea2751ea64b61c9f824420610a748cc1ddb","unresolved":false,"context_lines":[{"line_number":463,"context_line":"                        del seg_dict[key]"},{"line_number":464,"context_line":"        except (TypeError, binascii.Error):"},{"line_number":465,"context_line":"            errors.append("},{"line_number":466,"context_line":"                \"Index %d: %s must be valid base64 encoded data\""},{"line_number":467,"context_line":"                % (seg_index, key)"},{"line_number":468,"context_line":"            )"},{"line_number":469,"context_line":"            continue"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_e7f6c663","line":466,"in_reply_to":"7f231b9d_b90f10e0","updated":"2017-08-02 19:14:53.000000000","message":"I\u0027m not sure I understand this comment. Can you clarify? The valid JSON error should only be raised if json.loads() fails...","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7248d9acedb973a025c700d88c22d69cf730e3ec","unresolved":false,"context_lines":[{"line_number":463,"context_line":"                        del seg_dict[key]"},{"line_number":464,"context_line":"        except (TypeError, binascii.Error):"},{"line_number":465,"context_line":"            errors.append("},{"line_number":466,"context_line":"                \"Index %d: %s must be valid base64 encoded data\""},{"line_number":467,"context_line":"                % (seg_index, key)"},{"line_number":468,"context_line":"            )"},{"line_number":469,"context_line":"            continue"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_75ce0083","line":466,"in_reply_to":"df3967d1_e7f6c663","updated":"2017-08-02 20:06:30.000000000","message":"I wonder if maybe the body was transfer-encoded and more than the max manifest size...?","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"779faedafa11b86424b041f4c63baf2a6ae21801","unresolved":false,"context_lines":[{"line_number":863,"context_line":"            # here before we calculate the segment length (_segment_length)"},{"line_number":864,"context_line":"            seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":865,"context_line":"            if seg_preamble:"},{"line_number":866,"context_line":"                seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":867,"context_line":""},{"line_number":868,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":869,"context_line":"            if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":24,"id":"7f231b9d_79d05820","line":866,"updated":"2017-06-13 07:47:36.000000000","message":"This *should* not raise an error because the manifest has been validated during upload, but I would prefer some fallback in case we miss some strange case where the decode of the pre-/postamble fails. Or maybe I\u0027m simply to pessimistic?","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"5b6edea2751ea64b61c9f824420610a748cc1ddb","unresolved":false,"context_lines":[{"line_number":863,"context_line":"            # here before we calculate the segment length (_segment_length)"},{"line_number":864,"context_line":"            seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":865,"context_line":"            if seg_preamble:"},{"line_number":866,"context_line":"                seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":867,"context_line":""},{"line_number":868,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":869,"context_line":"            if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_07642a49","line":866,"in_reply_to":"7f231b9d_79d05820","updated":"2017-08-02 19:14:53.000000000","message":"Yeah, this seems reasonable.","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"ffdedb57d90f296625a8100f60006a0b3749aa80","unresolved":false,"context_lines":[{"line_number":863,"context_line":"            # here before we calculate the segment length (_segment_length)"},{"line_number":864,"context_line":"            seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":865,"context_line":"            if seg_preamble:"},{"line_number":866,"context_line":"                seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":867,"context_line":""},{"line_number":868,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":869,"context_line":"            if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_6a93bb5c","line":866,"in_reply_to":"df3967d1_07642a49","updated":"2017-08-02 19:32:33.000000000","message":"The problem is, what do we do in the case that data validated on upload is no longer valid when we download? Does kind of *feel* like a 500 :S","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7248d9acedb973a025c700d88c22d69cf730e3ec","unresolved":false,"context_lines":[{"line_number":863,"context_line":"            # here before we calculate the segment length (_segment_length)"},{"line_number":864,"context_line":"            seg_preamble \u003d seg_dict.get(\u0027preamble\u0027)"},{"line_number":865,"context_line":"            if seg_preamble:"},{"line_number":866,"context_line":"                seg_dict[\u0027preamble\u0027] \u003d base64.b64decode(seg_preamble)"},{"line_number":867,"context_line":""},{"line_number":868,"context_line":"            seg_postamble \u003d seg_dict.get(\u0027postamble\u0027)"},{"line_number":869,"context_line":"            if seg_postamble:"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_f5955034","line":866,"in_reply_to":"df3967d1_6a93bb5c","updated":"2017-08-02 20:06:30.000000000","message":"Or a 409, like should happen if a segment is inaccessible or doesn\u0027t match the description stored in the manifest...","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"779faedafa11b86424b041f4c63baf2a6ae21801","unresolved":false,"context_lines":[{"line_number":918,"context_line":"            end \u003d seg_dict.get(\u0027end_byte\u0027, int(seg_dict[\u0027bytes\u0027]) - 1)"},{"line_number":919,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"},{"line_number":920,"context_line":"            return is_small"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"        if req.range:"},{"line_number":923,"context_line":"            byteranges \u003d ["},{"line_number":924,"context_line":"                # For some reason, swob.Range.ranges_for_length adds 1 to the"}],"source_content_type":"text/x-python","patch_set":24,"id":"7f231b9d_59791c2d","line":921,"updated":"2017-06-13 07:47:36.000000000","message":"This block just moved, right? There is no specific reason for this, or am I missing sth here?","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"ffdedb57d90f296625a8100f60006a0b3749aa80","unresolved":false,"context_lines":[{"line_number":918,"context_line":"            end \u003d seg_dict.get(\u0027end_byte\u0027, int(seg_dict[\u0027bytes\u0027]) - 1)"},{"line_number":919,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"},{"line_number":920,"context_line":"            return is_small"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"        if req.range:"},{"line_number":923,"context_line":"            byteranges \u003d ["},{"line_number":924,"context_line":"                # For some reason, swob.Range.ranges_for_length adds 1 to the"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_cadaa749","line":921,"in_reply_to":"7f231b9d_59791c2d","updated":"2017-08-02 19:32:33.000000000","message":"Done","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"5b6edea2751ea64b61c9f824420610a748cc1ddb","unresolved":false,"context_lines":[{"line_number":918,"context_line":"            end \u003d seg_dict.get(\u0027end_byte\u0027, int(seg_dict[\u0027bytes\u0027]) - 1)"},{"line_number":919,"context_line":"            is_small \u003d (end - start + 1) \u003c self.slo.rate_limit_under_size"},{"line_number":920,"context_line":"            return is_small"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"        if req.range:"},{"line_number":923,"context_line":"            byteranges \u003d ["},{"line_number":924,"context_line":"                # For some reason, swob.Range.ranges_for_length adds 1 to the"}],"source_content_type":"text/x-python","patch_set":24,"id":"df3967d1_c759828e","line":921,"in_reply_to":"7f231b9d_59791c2d","updated":"2017-08-02 19:14:53.000000000","message":"Yes, this is just a simple move.","commit_id":"7762a37d67211c78da5252a01e40e9fce69678e6"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"884c0a7f75d884ea02705f194fb5cb3aef2c656f","unresolved":false,"context_lines":[{"line_number":320,"context_line":"SYSMETA_SLO_SIZE \u003d get_sys_meta_prefix(\u0027object\u0027) + \u0027slo-size\u0027"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"def parse_and_validate_input(req_body, req_path):"},{"line_number":324,"context_line":"    \"\"\""},{"line_number":325,"context_line":"    Given a request body, parses it and returns a list of dictionaries."},{"line_number":326,"context_line":""}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_190f6d2f","line":323,"range":{"start_line":323,"start_character":4,"end_line":323,"end_character":28},"updated":"2017-10-18 15:53:53.000000000","message":"I\u0027d really like to see some validation in here to check that we\u0027re not allowing a manifest that **only** contains data segments...\n\nOr to put that another way, even though we\u0027re allowing users to wrap objects with extra bits of data they don\u0027t want as objects, let\u0027s make sure they\u0027re still referencing a \u0027real\u0027 object too to preserve the spirit of the *LO.\n\nOh, but I have no objection to multiple data segments in sequence because it might actually make more sense to do that when you\u0027re creating the data :)","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d8a7e3237f93f96d8c39de4134516d27e0049d3a","unresolved":false,"context_lines":[{"line_number":320,"context_line":"SYSMETA_SLO_SIZE \u003d get_sys_meta_prefix(\u0027object\u0027) + \u0027slo-size\u0027"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"def parse_and_validate_input(req_body, req_path):"},{"line_number":324,"context_line":"    \"\"\""},{"line_number":325,"context_line":"    Given a request body, parses it and returns a list of dictionaries."},{"line_number":326,"context_line":""}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_03bb5ae2","line":323,"range":{"start_line":323,"start_character":4,"end_line":323,"end_character":28},"in_reply_to":"3f4b6375_190f6d2f","updated":"2017-10-19 17:28:58.000000000","message":"Toward the end, currently L463:\n\n if parsed_data and all(\u0027data\u0027 in d for d in parsed_data):\n     errors.append(\"Inline data segments require at least one \"\n                   \"object-backed segment.\")\n\nNote that we *currently* support manifests like\n\n []\n\nso it can\u0027t just be\n\n if all(...):\n     ...","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"884c0a7f75d884ea02705f194fb5cb3aef2c656f","unresolved":false,"context_lines":[{"line_number":369,"context_line":"            (required for required in SLO_KEYS"},{"line_number":370,"context_line":"             if all(k in seg_dict for k in required)),"},{"line_number":371,"context_line":"            None)"},{"line_number":372,"context_line":"        if not segment_type:"},{"line_number":373,"context_line":"            errors.append("},{"line_number":374,"context_line":"                \"Index %d: expected keys to be supersets of one of %s\""},{"line_number":375,"context_line":"                % (seg_index,"}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_76b1cee1","line":372,"range":{"start_line":372,"start_character":8,"end_line":372,"end_character":28},"updated":"2017-10-18 15:53:53.000000000","message":"Feels like we need a `continue` if we fail to discover a valid segment type.","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d8a7e3237f93f96d8c39de4134516d27e0049d3a","unresolved":false,"context_lines":[{"line_number":369,"context_line":"            (required for required in SLO_KEYS"},{"line_number":370,"context_line":"             if all(k in seg_dict for k in required)),"},{"line_number":371,"context_line":"            None)"},{"line_number":372,"context_line":"        if not segment_type:"},{"line_number":373,"context_line":"            errors.append("},{"line_number":374,"context_line":"                \"Index %d: expected keys to be supersets of one of %s\""},{"line_number":375,"context_line":"                % (seg_index,"}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_a31b6619","line":372,"range":{"start_line":372,"start_character":8,"end_line":372,"end_character":28},"in_reply_to":"3f4b6375_76b1cee1","updated":"2017-10-19 17:28:58.000000000","message":"Good call.","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d8a7e3237f93f96d8c39de4134516d27e0049d3a","unresolved":false,"context_lines":[{"line_number":1119,"context_line":"            last_modified_formatted \u003d last_modified.strftime("},{"line_number":1120,"context_line":"                \u0027%Y-%m-%dT%H:%M:%S.%f\u0027"},{"line_number":1121,"context_line":"            )"},{"line_number":1122,"context_line":"            seg_data \u003d{"},{"line_number":1123,"context_line":"                \u0027name\u0027: \u0027/\u0027 + seg_dict[\u0027path\u0027].lstrip(\u0027/\u0027),"},{"line_number":1124,"context_line":"                \u0027bytes\u0027: head_seg_resp.content_length,"},{"line_number":1125,"context_line":"                \u0027hash\u0027: head_seg_resp.etag,"}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_6360cea6","line":1122,"range":{"start_line":1122,"start_character":20,"end_line":1122,"end_character":23},"updated":"2017-10-19 17:28:58.000000000","message":"*grumble, grumble*","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"00b23010d594a3887637098c47ea8bbe87c915e6","unresolved":false,"context_lines":[{"line_number":135,"context_line":"    echo -n \u0027etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;\u0027 \\"},{"line_number":136,"context_line":"    | md5sum"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"For the purposes of Etag computations, inlined data segments are consired to"},{"line_number":139,"context_line":"have an etag of the md5 of the raw data (i.e., *not* base64-encoded)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f4b6375_13a2644c","line":138,"range":{"start_line":138,"start_character":65,"end_line":138,"end_character":73},"updated":"2018-01-30 19:38:33.000000000","message":"*considered","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d5afbfd7dbbc32014b658545d414be298a769960","unresolved":false,"context_lines":[{"line_number":135,"context_line":"    echo -n \u0027etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;\u0027 \\"},{"line_number":136,"context_line":"    | md5sum"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"For the purposes of Etag computations, inlined data segments are consired to"},{"line_number":139,"context_line":"have an etag of the md5 of the raw data (i.e., *not* base64-encoded)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"5f93b717_9d8c0880","line":138,"range":{"start_line":138,"start_character":65,"end_line":138,"end_character":73},"in_reply_to":"3f4b6375_13a2644c","updated":"2018-01-30 21:42:00.000000000","message":"Done ;-)","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0b681f907eaf1a5c04e9226be59709f7958b4fc8","unresolved":false,"context_lines":[{"line_number":1036,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1037,"context_line":"                \u0027Number of object-backed segments must be \u003c\u003d %d\u0027 %"},{"line_number":1038,"context_line":"                self.max_manifest_segments)"},{"line_number":1039,"context_line":"        if sum(1 for seg in parsed_data if \u0027data\u0027 in seg) \u003e \\"},{"line_number":1040,"context_line":"                self.max_manifest_segments:"},{"line_number":1041,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1042,"context_line":"                \u0027Number of inlined-data segments must be \u003c\u003d %d\u0027 %"},{"line_number":1043,"context_line":"                self.max_manifest_segments)"},{"line_number":1044,"context_line":"        total_size \u003d 0"},{"line_number":1045,"context_line":"        try:"},{"line_number":1046,"context_line":"            out_content_type \u003d req.accept.best_match(ACCEPTABLE_FORMATS)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f4b6375_4e8f3923","line":1043,"range":{"start_line":1039,"start_character":8,"end_line":1043,"end_character":43},"updated":"2017-10-19 18:05:12.000000000","message":"I also considered just dropping this check, and saying you could have as many data segments as you like -- we\u0027re only really concerned about the number of segments because of all the HEADs it will spawn and how long it\u0027ll take us to get a response back to the client, right?\n\nOtherwise, we can just rely on the max_manifest_size to keep memory usage in check.","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"6a4164c0f9fea2f401cf7d888f3a10f6938793bc","unresolved":false,"context_lines":[{"line_number":1036,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1037,"context_line":"                \u0027Number of object-backed segments must be \u003c\u003d %d\u0027 %"},{"line_number":1038,"context_line":"                self.max_manifest_segments)"},{"line_number":1039,"context_line":"        if sum(1 for seg in parsed_data if \u0027data\u0027 in seg) \u003e \\"},{"line_number":1040,"context_line":"                self.max_manifest_segments:"},{"line_number":1041,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1042,"context_line":"                \u0027Number of inlined-data segments must be \u003c\u003d %d\u0027 %"},{"line_number":1043,"context_line":"                self.max_manifest_segments)"},{"line_number":1044,"context_line":"        total_size \u003d 0"},{"line_number":1045,"context_line":"        try:"},{"line_number":1046,"context_line":"            out_content_type \u003d req.accept.best_match(ACCEPTABLE_FORMATS)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f4b6375_6d02c6b6","line":1043,"range":{"start_line":1039,"start_character":8,"end_line":1043,"end_character":43},"in_reply_to":"3f4b6375_4e8f3923","updated":"2017-11-01 12:09:40.000000000","message":"Yeah, I\u0027m not convinced that we need to count the number of data segments. In the initial patch I raised the max manifest size to cope with the potential extra data. The data segments are all in memory so that\u0027s the thing we need to focus on controlling, the count is largely irrelevant due to the lack of HEADs.","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"24aa8acab3c94b07f717afe5736eefd63bf9246d","unresolved":false,"context_lines":[{"line_number":1036,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1037,"context_line":"                \u0027Number of object-backed segments must be \u003c\u003d %d\u0027 %"},{"line_number":1038,"context_line":"                self.max_manifest_segments)"},{"line_number":1039,"context_line":"        if sum(1 for seg in parsed_data if \u0027data\u0027 in seg) \u003e \\"},{"line_number":1040,"context_line":"                self.max_manifest_segments:"},{"line_number":1041,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1042,"context_line":"                \u0027Number of inlined-data segments must be \u003c\u003d %d\u0027 %"},{"line_number":1043,"context_line":"                self.max_manifest_segments)"},{"line_number":1044,"context_line":"        total_size \u003d 0"},{"line_number":1045,"context_line":"        try:"},{"line_number":1046,"context_line":"            out_content_type \u003d req.accept.best_match(ACCEPTABLE_FORMATS)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f4b6375_2c02820c","line":1043,"range":{"start_line":1039,"start_character":8,"end_line":1043,"end_character":43},"in_reply_to":"3f4b6375_6d02c6b6","updated":"2017-11-01 21:23:09.000000000","message":"Done","commit_id":"a93117019e7a9bbc654e0fda975b04d4993ef11d"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":164,"context_line":"    echo -n \u0027etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;\u0027 \\"},{"line_number":165,"context_line":"    | md5sum"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"For the purposes of Etag computations, inlined data segments are consired to"},{"line_number":168,"context_line":"have an etag of the md5 of the raw data (i.e., *not* base64-encoded)."},{"line_number":169,"context_line":""},{"line_number":170,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_7cc34139","line":167,"range":{"start_line":167,"start_character":65,"end_line":167,"end_character":73},"updated":"2017-11-04 12:13:27.000000000","message":"considered ?","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":164,"context_line":"    echo -n \u0027etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;\u0027 \\"},{"line_number":165,"context_line":"    | md5sum"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"For the purposes of Etag computations, inlined data segments are consired to"},{"line_number":168,"context_line":"have an etag of the md5 of the raw data (i.e., *not* base64-encoded)."},{"line_number":169,"context_line":""},{"line_number":170,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_64526de0","line":167,"range":{"start_line":167,"start_character":65,"end_line":167,"end_character":73},"in_reply_to":"3f4b6375_7cc34139","updated":"2017-11-06 20:10:42.000000000","message":"Done","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":204,"context_line":"Inline Data Specification"},{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_3cdf090e","line":207,"range":{"start_line":207,"start_character":27,"end_line":207,"end_character":33},"updated":"2017-11-04 12:13:27.000000000","message":"users","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":204,"context_line":"Inline Data Specification"},{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_246235af","line":207,"range":{"start_line":207,"start_character":27,"end_line":207,"end_character":33},"in_reply_to":"3f4b6375_3cdf090e","updated":"2017-11-06 20:10:42.000000000","message":"Done","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":211,"context_line":"referenced as separate objects."},{"line_number":212,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_dce7d5e2","line":209,"range":{"start_line":208,"start_character":56,"end_line":209,"end_character":29},"updated":"2017-11-04 12:13:27.000000000","message":"Is \"should be\" correct? If \"base64-encoded\" is strong restriction, I think \"must be\" is better.\nBut I don\u0027t know why the inline data should be base64-encoded. Are there any reason for encoding?","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":211,"context_line":"referenced as separate objects."},{"line_number":212,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_7cbc28ba","line":209,"range":{"start_line":208,"start_character":56,"end_line":209,"end_character":29},"in_reply_to":"3f4b6375_ceaf0d7b","updated":"2017-11-08 07:31:43.000000000","message":"Tim, Joel, thank you for your answer, I understand reason.","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":211,"context_line":"referenced as separate objects."},{"line_number":212,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_0465f9b4","line":209,"range":{"start_line":208,"start_character":56,"end_line":209,"end_character":29},"in_reply_to":"3f4b6375_dce7d5e2","updated":"2017-11-06 20:10:42.000000000","message":"Done.\n\nIt needs to be encoded so that we can include bytes that are not valid UTF-8 (like b\u0027\\xff\u0027). JSON requires that all strings be unicode, and it seemed confusing to use UTF-8 for things like \u0027path\u0027 but use something like ISO-8859-1 for \u0027data\u0027, even if it would map well with bytes.\n\nThere\u0027s also a benefit for non-printable characters that *are* valid UTF-8 -- the null byte (which is used as padding in tar files), would expand 6x to\n\n \\u0000","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"a5254704629441119610dc18f65b416d4bd09006","unresolved":false,"context_lines":[{"line_number":205,"context_line":"-------------------------"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"When uploading a manifest, uesers can include \u0027data\u0027 segments that should"},{"line_number":208,"context_line":"be included between objects. The data in these segments should"},{"line_number":209,"context_line":"be base64-encoded binary data and will be included in the etag of the"},{"line_number":210,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":211,"context_line":"referenced as separate objects."},{"line_number":212,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_ceaf0d7b","line":209,"range":{"start_line":208,"start_character":56,"end_line":209,"end_character":29},"in_reply_to":"3f4b6375_dce7d5e2","updated":"2017-11-07 21:57:41.000000000","message":"The data must be base64-encoded because the manifest must be a valid JSON object and we need to be able to represent arbitrary binary data in a JSON string.","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":344,"context_line":""},{"line_number":345,"context_line":"SLO_KEYS \u003d {"},{"line_number":346,"context_line":"    # required: optional"},{"line_number":347,"context_line":"    (\u0027data\u0027, ): set(),"},{"line_number":348,"context_line":"    (\u0027path\u0027, ): {\u0027range\u0027, \u0027etag\u0027, \u0027size_bytes\u0027},"},{"line_number":349,"context_line":"}"},{"line_number":350,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_fc0bb179","line":347,"range":{"start_line":347,"start_character":4,"end_line":347,"end_character":14},"updated":"2017-11-04 12:13:27.000000000","message":"Why is this tupled?","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":344,"context_line":""},{"line_number":345,"context_line":"SLO_KEYS \u003d {"},{"line_number":346,"context_line":"    # required: optional"},{"line_number":347,"context_line":"    (\u0027data\u0027, ): set(),"},{"line_number":348,"context_line":"    (\u0027path\u0027, ): {\u0027range\u0027, \u0027etag\u0027, \u0027size_bytes\u0027},"},{"line_number":349,"context_line":"}"},{"line_number":350,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_84e32905","line":347,"range":{"start_line":347,"start_character":4,"end_line":347,"end_character":14},"in_reply_to":"3f4b6375_fc0bb179","updated":"2017-11-06 20:10:42.000000000","message":"Maybe it\u0027s not necessary. I was thinking about how (before https://github.com/openstack/swift/commit/7fb102d), this would have needed something like\n\n SLO_KEYS \u003d {\n     (\u0027data\u0027, ): set(),\n     (\u0027path\u0027, \u0027etag\u0027, \u0027size_bytes\u0027): {\u0027range\u0027},\n }","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":397,"context_line":"            errors.append(\"Index %d: not a JSON object\" % seg_index)"},{"line_number":398,"context_line":"            continue"},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"        segment_type \u003d next("},{"line_number":401,"context_line":"            (required for required in SLO_KEYS"},{"line_number":402,"context_line":"             if all(k in seg_dict for k in required)),"},{"line_number":403,"context_line":"            None)"},{"line_number":404,"context_line":"        if not segment_type:"},{"line_number":405,"context_line":"            errors.append("},{"line_number":406,"context_line":"                \"Index %d: expected keys to be supersets of one of %s\""},{"line_number":407,"context_line":"                % (seg_index,"},{"line_number":408,"context_line":"                   \" or \".join(repr(set(required)) for required in SLO_KEYS)))"},{"line_number":409,"context_line":"            continue"},{"line_number":410,"context_line":""},{"line_number":411,"context_line":"        allowed_keys \u003d SLO_KEYS[segment_type].union(segment_type)"},{"line_number":412,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_5cec05ba","line":409,"range":{"start_line":400,"start_character":8,"end_line":409,"end_character":20},"updated":"2017-11-04 12:13:27.000000000","message":"I feel this code is confusing....\n\nHow about following code?\n\ntry:\n    segment_type \u003d (set(SLO_KEYS).keys() \u0026 set(seg_dict.keys())).pop()\nexcept KeyError:\n    errors.append(\n        \"Index %d: expected keys to be supersets of one of %s\"\n         % (seg_index,\n             \" or \".join(repr(set(required)) for required in SLO_KEYS)))\n    continue","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":397,"context_line":"            errors.append(\"Index %d: not a JSON object\" % seg_index)"},{"line_number":398,"context_line":"            continue"},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"        segment_type \u003d next("},{"line_number":401,"context_line":"            (required for required in SLO_KEYS"},{"line_number":402,"context_line":"             if all(k in seg_dict for k in required)),"},{"line_number":403,"context_line":"            None)"},{"line_number":404,"context_line":"        if not segment_type:"},{"line_number":405,"context_line":"            errors.append("},{"line_number":406,"context_line":"                \"Index %d: expected keys to be supersets of one of %s\""},{"line_number":407,"context_line":"                % (seg_index,"},{"line_number":408,"context_line":"                   \" or \".join(repr(set(required)) for required in SLO_KEYS)))"},{"line_number":409,"context_line":"            continue"},{"line_number":410,"context_line":""},{"line_number":411,"context_line":"        allowed_keys \u003d SLO_KEYS[segment_type].union(segment_type)"},{"line_number":412,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_a4d2a5fe","line":409,"range":{"start_line":400,"start_character":8,"end_line":409,"end_character":20},"in_reply_to":"3f4b6375_5cec05ba","updated":"2017-11-06 20:10:42.000000000","message":"Or maybe a for/else loop? Or I should just take it as a sign -- if we don\u0027t currently *need* multiple required keys, and we\u0027ve been moving away from needing them, maybe I should just drop it for now.","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"e7e0809e38f5b3f9578143e240ad6ef1f48c8027","unresolved":false,"context_lines":[{"line_number":494,"context_line":"            seg_dict[\u0027data\u0027] \u003d base64.b64encode(data)"},{"line_number":495,"context_line":""},{"line_number":496,"context_line":"    if parsed_data and all(\u0027data\u0027 in d for d in parsed_data):"},{"line_number":497,"context_line":"        errors.append(\"Inline data segments require at least one \""},{"line_number":498,"context_line":"                      \"object-backed segment.\")"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"    if errors:"},{"line_number":501,"context_line":"        error_message \u003d \"\".join(e + \"\\n\" for e in errors)"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_1c19edd5","line":498,"range":{"start_line":497,"start_character":7,"end_line":498,"end_character":47},"updated":"2017-11-04 12:13:27.000000000","message":"Is this restriction explained in head of slo.py?\nI didn\u0027t find explanation for this restriction.","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bf502a4543a290518ac67223c9dcda8fd42ef8ab","unresolved":false,"context_lines":[{"line_number":494,"context_line":"            seg_dict[\u0027data\u0027] \u003d base64.b64encode(data)"},{"line_number":495,"context_line":""},{"line_number":496,"context_line":"    if parsed_data and all(\u0027data\u0027 in d for d in parsed_data):"},{"line_number":497,"context_line":"        errors.append(\"Inline data segments require at least one \""},{"line_number":498,"context_line":"                      \"object-backed segment.\")"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"    if errors:"},{"line_number":501,"context_line":"        error_message \u003d \"\".join(e + \"\\n\" for e in errors)"}],"source_content_type":"text/x-python","patch_set":34,"id":"3f4b6375_44977146","line":498,"range":{"start_line":497,"start_character":7,"end_line":498,"end_character":47},"in_reply_to":"3f4b6375_1c19edd5","updated":"2017-11-06 20:10:42.000000000","message":"In \"Uploading the Manifest\", just after the (short) table for data segments:\n\n\u003e At least one object-backed segment must be included.\n\nWe could call this out more.","commit_id":"8b8475dfc65aff89a58a4b44da1d42dbecd02eb0"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"a5254704629441119610dc18f65b416d4bd09006","unresolved":false,"context_lines":[{"line_number":402,"context_line":"            continue"},{"line_number":403,"context_line":""},{"line_number":404,"context_line":"        for required in SLO_KEYS:"},{"line_number":405,"context_line":"            if required in seg_dict:"},{"line_number":406,"context_line":"                segment_type \u003d required"},{"line_number":407,"context_line":"                break"},{"line_number":408,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_6796d640","line":405,"range":{"start_line":405,"start_character":0,"end_line":405,"end_character":12},"updated":"2017-11-07 21:57:41.000000000","message":"It occurs to me that I can\u0027t see anything in here that would stop us doing something dumb like specifying a \u0027path\u0027 segment that contains a \u0027data\u0027 key. I\u0027m also not sure what type of segment we\u0027d end up with in this case...","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f5a6c0a00cb8c16369871fbba91dac71005a7e59","unresolved":false,"context_lines":[{"line_number":402,"context_line":"            continue"},{"line_number":403,"context_line":""},{"line_number":404,"context_line":"        for required in SLO_KEYS:"},{"line_number":405,"context_line":"            if required in seg_dict:"},{"line_number":406,"context_line":"                segment_type \u003d required"},{"line_number":407,"context_line":"                break"},{"line_number":408,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_073ba237","line":405,"range":{"start_line":405,"start_character":0,"end_line":405,"end_character":12},"in_reply_to":"3f4b6375_6796d640","updated":"2017-11-07 22:01:26.000000000","message":"So segment_type could be either path or data, but whichever one it\u0027s *not* will show up in extraneous_keys below. Might be good to add a test to demonstrate it.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":412,"context_line":"                   \" or \".join(repr(required) for required in SLO_KEYS)))"},{"line_number":413,"context_line":"            continue"},{"line_number":414,"context_line":""},{"line_number":415,"context_line":"        allowed_keys \u003d SLO_KEYS[segment_type].union([segment_type])"},{"line_number":416,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"},{"line_number":417,"context_line":"        if extraneous_keys:"},{"line_number":418,"context_line":"            errors.append("}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_bcf50053","line":415,"range":{"start_line":415,"start_character":45,"end_line":415,"end_character":67},"updated":"2017-11-08 07:31:43.000000000","message":"\".add(segment_type)\" is more simple.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":544,"context_line":"                \u0027while fetching %s, JSON-decoding of submanifest %s \u0027"},{"line_number":545,"context_line":"                \u0027failed with %s\u0027 % (req.path, sub_req.path, err))"},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"    def _segment_path(self, version, account, seg_dict):"},{"line_number":548,"context_line":"        return \"/{ver}/{acc}/{conobj}\".format("},{"line_number":549,"context_line":"            ver\u003dversion, acc\u003daccount,"},{"line_number":550,"context_line":"            conobj\u003dseg_dict[\u0027name\u0027].lstrip(\u0027/\u0027)"},{"line_number":551,"context_line":"        )"},{"line_number":552,"context_line":""},{"line_number":553,"context_line":"    def _segment_length(self, seg_dict):"},{"line_number":554,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_0fcdf864","line":551,"range":{"start_line":547,"start_character":0,"end_line":551,"end_character":9},"updated":"2017-11-08 07:31:43.000000000","message":"Is this function used?","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":555,"context_line":"        Returns the number of bytes that will be fetched from the specified"},{"line_number":556,"context_line":"        segment on a plain GET request for this SLO manifest."},{"line_number":557,"context_line":"        \"\"\""},{"line_number":558,"context_line":"        if \u0027data\u0027 in seg_dict:"},{"line_number":559,"context_line":"            # NB: should already be decoded!"},{"line_number":560,"context_line":"            return len(seg_dict[\u0027data\u0027])"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"        seg_range \u003d seg_dict.get(\u0027range\u0027)"},{"line_number":563,"context_line":"        if seg_range is not None:"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_8d8b301d","line":560,"range":{"start_line":558,"start_character":1,"end_line":560,"end_character":40},"updated":"2017-11-08 07:31:43.000000000","message":"As I wrote in get_or_head_response method, I think variable meanings should not be changed.\nWith this code, all caller of this method have to check \u0027data\u0027 of seg_dict is decoded.\nI think this code is dangerous because this method will return wrong length without error, if the seg_dict has non-decoded \u0027data\u0027.\n\nSo my opinion is that \"seg_dict should not be updated\" and \"_segment_length should decode seg_dict[\u0027data\u0027]\".","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":555,"context_line":"        Returns the number of bytes that will be fetched from the specified"},{"line_number":556,"context_line":"        segment on a plain GET request for this SLO manifest."},{"line_number":557,"context_line":"        \"\"\""},{"line_number":558,"context_line":"        if \u0027data\u0027 in seg_dict:"},{"line_number":559,"context_line":"            # NB: should already be decoded!"},{"line_number":560,"context_line":"            return len(seg_dict[\u0027data\u0027])"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"        seg_range \u003d seg_dict.get(\u0027range\u0027)"},{"line_number":563,"context_line":"        if seg_range is not None:"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_b44205aa","line":560,"range":{"start_line":558,"start_character":1,"end_line":560,"end_character":40},"in_reply_to":"3f4b6375_8d8b301d","updated":"2017-11-16 16:44:23.000000000","message":"If we can guarantee that the data segments are decoded when the manifest is retrieved, why should we repeat the decoding work in multiple places? This seems wasteful.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":609,"context_line":"            for seg_info in byterange_listing_iter:"},{"line_number":610,"context_line":"                yield seg_info"},{"line_number":611,"context_line":""},{"line_number":612,"context_line":"    def _byterange_listing_iterator(self, req, version, account, segments,"},{"line_number":613,"context_line":"                                    first_byte, last_byte,"},{"line_number":614,"context_line":"                                    cached_fetch_sub_slo_segments,"},{"line_number":615,"context_line":"                                    recursion_depth\u003d1):"},{"line_number":616,"context_line":"        last_sub_path \u003d None"},{"line_number":617,"context_line":"        for seg_dict in segments:"},{"line_number":618,"context_line":"            seg_length \u003d self._segment_length(seg_dict)"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_4d11b85c","line":615,"range":{"start_line":612,"start_character":4,"end_line":615,"end_character":55},"updated":"2017-11-08 07:31:43.000000000","message":"Why first_byte and last_byte are included in seg_dict of yield value?\nI think the 2 variables are not segment members.\nThis change cause \"seg_dict witthout first_byte or last_byte\" and \"seg_dict with first_byte and last_byte\" in slo middleware. This is confusing a little.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":609,"context_line":"            for seg_info in byterange_listing_iter:"},{"line_number":610,"context_line":"                yield seg_info"},{"line_number":611,"context_line":""},{"line_number":612,"context_line":"    def _byterange_listing_iterator(self, req, version, account, segments,"},{"line_number":613,"context_line":"                                    first_byte, last_byte,"},{"line_number":614,"context_line":"                                    cached_fetch_sub_slo_segments,"},{"line_number":615,"context_line":"                                    recursion_depth\u003d1):"},{"line_number":616,"context_line":"        last_sub_path \u003d None"},{"line_number":617,"context_line":"        for seg_dict in segments:"},{"line_number":618,"context_line":"            seg_length \u003d self._segment_length(seg_dict)"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_14fb1979","line":615,"range":{"start_line":612,"start_character":4,"end_line":615,"end_character":55},"in_reply_to":"3f4b6375_4d11b85c","updated":"2017-11-16 16:44:23.000000000","message":"I think the \u0027first_byte\u0027 and \u0027last_byte\u0027 here are \u0027first_slo_byte\u0027 and \u0027last_slo_byte\u0027 not \u0027first_segment_byte\u0027 and \u0027last_segment_byte\u0027...\n\nMaybe it\u0027s worth renaming them?","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":883,"context_line":""},{"line_number":884,"context_line":"        # Decode any inlined data, and calculate content_length \u0026 etag"},{"line_number":885,"context_line":"        # if necessary"},{"line_number":886,"context_line":"        if slo_etag is None:"},{"line_number":887,"context_line":"            calculated_etag \u003d md5()"},{"line_number":888,"context_line":"        if content_length is None:"},{"line_number":889,"context_line":"            calculated_content_length \u003d 0"},{"line_number":890,"context_line":""},{"line_number":891,"context_line":"        for seg_dict in segments:"},{"line_number":892,"context_line":"            # It\u0027s important that we decode the data *before*"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_8d00d002","line":889,"range":{"start_line":886,"start_character":0,"end_line":889,"end_character":41},"updated":"2017-11-08 07:31:43.000000000","message":"Why do you split this checking?\nIn my understainging, SYSMETA_SLO_ETAG and SYSMETA_SLO_SIZE are added together by slo middleware.\nSo I think this splitting is not effective.\n\nI think change for this method should be only adding etag and size update for inline data segments to the following for loop.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":891,"context_line":"        for seg_dict in segments:"},{"line_number":892,"context_line":"            # It\u0027s important that we decode the data *before*"},{"line_number":893,"context_line":"            # calculating the segment length and etag"},{"line_number":894,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":895,"context_line":"                seg_dict[\u0027data\u0027] \u003d base64.b64decode(seg_dict[\u0027data\u0027])"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"            if slo_etag is None:"},{"line_number":898,"context_line":"                if \u0027data\u0027 in seg_dict:"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_2da55c03","line":895,"range":{"start_line":894,"start_character":1,"end_line":895,"end_character":69},"updated":"2017-11-08 07:31:43.000000000","message":"This code changes meanings of \"seg_dict\" and this is confusing a little.\n\nHow about following code?:\n\ninline_data \u003d None\nif \u0027data\u0027 in seg_dict:\n    inline_data \u003d base64.b64decode(seg_dict[\u0027data\u0027])","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"52838d5f8799354a2b194132b1c5f8291d66a9c9","unresolved":false,"context_lines":[{"line_number":891,"context_line":"        for seg_dict in segments:"},{"line_number":892,"context_line":"            # It\u0027s important that we decode the data *before*"},{"line_number":893,"context_line":"            # calculating the segment length and etag"},{"line_number":894,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":895,"context_line":"                seg_dict[\u0027data\u0027] \u003d base64.b64decode(seg_dict[\u0027data\u0027])"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"            if slo_etag is None:"},{"line_number":898,"context_line":"                if \u0027data\u0027 in seg_dict:"}],"source_content_type":"text/x-python","patch_set":36,"id":"df87a7cf_0bd4e5e7","line":895,"range":{"start_line":894,"start_character":1,"end_line":895,"end_character":69},"in_reply_to":"1f485f77_54af517c","updated":"2017-12-11 19:57:27.000000000","message":"This was a solid code-smell -- good call Kazuhiro! I think there may be an issue with data segments in nested SLOs. Currently, we do the decode() here in get_or_head_response, but we never do it for sub-SLOs.\n\nMaybe it\u0027d be better to switch things over to another namespace here, like\n\n if \u0027data\u0027 in seg_dict:\n     seg_dict[\u0027raw_data\u0027] \u003d base64decode(seg_dict.pop(\u0027data\u0027))\n\nand do something similar toward the start of the loop in _byterange_listing_iterator.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":891,"context_line":"        for seg_dict in segments:"},{"line_number":892,"context_line":"            # It\u0027s important that we decode the data *before*"},{"line_number":893,"context_line":"            # calculating the segment length and etag"},{"line_number":894,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":895,"context_line":"                seg_dict[\u0027data\u0027] \u003d base64.b64decode(seg_dict[\u0027data\u0027])"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"            if slo_etag is None:"},{"line_number":898,"context_line":"                if \u0027data\u0027 in seg_dict:"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_54af517c","line":895,"range":{"start_line":894,"start_character":1,"end_line":895,"end_character":69},"in_reply_to":"3f4b6375_2da55c03","updated":"2017-11-16 16:44:23.000000000","message":"The in-place update is important here - we later rely on the data having been decoded","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":895,"context_line":"                seg_dict[\u0027data\u0027] \u003d base64.b64decode(seg_dict[\u0027data\u0027])"},{"line_number":896,"context_line":""},{"line_number":897,"context_line":"            if slo_etag is None:"},{"line_number":898,"context_line":"                if \u0027data\u0027 in seg_dict:"},{"line_number":899,"context_line":"                    calculated_etag.update(md5(seg_dict[\u0027data\u0027]).hexdigest())"},{"line_number":900,"context_line":"                elif seg_dict.get(\u0027range\u0027):"},{"line_number":901,"context_line":"                    calculated_etag.update("},{"line_number":902,"context_line":"                        \u0027%s:%s;\u0027 % (seg_dict[\u0027hash\u0027], seg_dict[\u0027range\u0027])"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_8dd33060","line":899,"range":{"start_line":898,"start_character":0,"end_line":899,"end_character":77},"updated":"2017-11-08 07:31:43.000000000","message":"This code is updated to following code in the above idea.:\n\nif inline_data:\n    calclulated_etag.update(md5(inline_data).hexdigest())","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":943,"context_line":"        plain_listing_iter \u003d self._segment_listing_iterator("},{"line_number":944,"context_line":"            req, ver, account, segments, byteranges)"},{"line_number":945,"context_line":""},{"line_number":946,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":947,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":948,"context_line":"                return False  # it\u0027s already in memory anyway"},{"line_number":949,"context_line":"            start \u003d seg_dict.get(\u0027start_byte\u0027) or 0"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_4881a6fa","line":946,"range":{"start_line":946,"start_character":12,"end_line":946,"end_character":28},"updated":"2017-11-08 07:31:43.000000000","message":"I agree that inline data is not ratelimited.\nBut this function name is not match the implementation now because the inline data feature is aimed at tiny objects.\nI think the function name should be updated. (e.g. \"is_small_object_backed_segment\")","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"52838d5f8799354a2b194132b1c5f8291d66a9c9","unresolved":false,"context_lines":[{"line_number":943,"context_line":"        plain_listing_iter \u003d self._segment_listing_iterator("},{"line_number":944,"context_line":"            req, ver, account, segments, byteranges)"},{"line_number":945,"context_line":""},{"line_number":946,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":947,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":948,"context_line":"                return False  # it\u0027s already in memory anyway"},{"line_number":949,"context_line":"            start \u003d seg_dict.get(\u0027start_byte\u0027) or 0"}],"source_content_type":"text/x-python","patch_set":36,"id":"df87a7cf_cb04bd17","line":946,"range":{"start_line":946,"start_character":12,"end_line":946,"end_character":28},"in_reply_to":"1f485f77_1457995a","updated":"2017-12-11 19:57:27.000000000","message":"Done","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":943,"context_line":"        plain_listing_iter \u003d self._segment_listing_iterator("},{"line_number":944,"context_line":"            req, ver, account, segments, byteranges)"},{"line_number":945,"context_line":""},{"line_number":946,"context_line":"        def is_small_segment(seg_dict):"},{"line_number":947,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":948,"context_line":"                return False  # it\u0027s already in memory anyway"},{"line_number":949,"context_line":"            start \u003d seg_dict.get(\u0027start_byte\u0027) or 0"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_1457995a","line":946,"range":{"start_line":946,"start_character":12,"end_line":946,"end_character":28},"in_reply_to":"3f4b6375_4881a6fa","updated":"2017-11-16 16:44:23.000000000","message":"+1","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"            req.path)"},{"line_number":1080,"context_line":"        problem_segments \u003d []"},{"line_number":1081,"context_line":""},{"line_number":1082,"context_line":"        if sum(1 for seg in parsed_data if \u0027path\u0027 in seg) \u003e \\"},{"line_number":1083,"context_line":"                self.max_manifest_segments:"},{"line_number":1084,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1085,"context_line":"                \u0027Number of object-backed segments must be \u003c\u003d %d\u0027 %"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_4859a68b","line":1082,"range":{"start_line":1082,"start_character":11,"end_line":1082,"end_character":57},"updated":"2017-11-08 07:31:43.000000000","message":"IMO, \"sum(\u0027path\u0027 in seg for seg in parsed_data)\" is more simple.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"            req.path)"},{"line_number":1080,"context_line":"        problem_segments \u003d []"},{"line_number":1081,"context_line":""},{"line_number":1082,"context_line":"        if sum(1 for seg in parsed_data if \u0027path\u0027 in seg) \u003e \\"},{"line_number":1083,"context_line":"                self.max_manifest_segments:"},{"line_number":1084,"context_line":"            raise HTTPRequestEntityTooLarge("},{"line_number":1085,"context_line":"                \u0027Number of object-backed segments must be \u003c\u003d %d\u0027 %"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_34adf59a","line":1082,"range":{"start_line":1082,"start_character":11,"end_line":1082,"end_character":57},"in_reply_to":"3f4b6375_4859a68b","updated":"2017-11-16 16:44:23.000000000","message":"Personally I prefer the transparency of calling out \u0027give me a 1 for every True\u0027 rather than implicitly using Python typecasting of bool -\u003e int.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":1144,"context_line":"                problem_segments.append("},{"line_number":1145,"context_line":"                    [quote(obj_name),"},{"line_number":1146,"context_line":"                     \u0027Too small; each segment must be at least 1 byte.\u0027])"},{"line_number":1147,"context_line":""},{"line_number":1148,"context_line":"            _size_bytes \u003d seg_dict.get(\u0027size_bytes\u0027)"},{"line_number":1149,"context_line":"            size_mismatch \u003d ("},{"line_number":1150,"context_line":"                _size_bytes is not None and"},{"line_number":1151,"context_line":"                _size_bytes !\u003d head_seg_resp.content_length"},{"line_number":1152,"context_line":"            )"},{"line_number":1153,"context_line":"            if size_mismatch:"},{"line_number":1154,"context_line":"                problem_segments.append([quote(obj_name), \u0027Size Mismatch\u0027])"},{"line_number":1155,"context_line":""},{"line_number":1156,"context_line":"            _etag \u003d seg_dict.get(\u0027etag\u0027)"},{"line_number":1157,"context_line":"            etag_mismatch \u003d ("},{"line_number":1158,"context_line":"                _etag is not None and"},{"line_number":1159,"context_line":"                _etag !\u003d head_seg_resp.etag"},{"line_number":1160,"context_line":"            )"},{"line_number":1161,"context_line":"            if etag_mismatch:"},{"line_number":1162,"context_line":"                problem_segments.append([quote(obj_name), \u0027Etag Mismatch\u0027])"},{"line_number":1163,"context_line":""},{"line_number":1164,"context_line":"            if head_seg_resp.last_modified:"},{"line_number":1165,"context_line":"                last_modified \u003d head_seg_resp.last_modified"}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_e30e3330","line":1162,"range":{"start_line":1147,"start_character":0,"end_line":1162,"end_character":75},"updated":"2017-11-08 07:31:43.000000000","message":"This is refactoring change and I like this change.\nBut I think this is not change for \"Add support for data segments to SLO and SegmentedIterable\".\nI think this change should be done by another patch.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"649d763d9651112e03332010db97f66c15f73034","unresolved":false,"context_lines":[{"line_number":1167,"context_line":"                # shouldn\u0027t happen"},{"line_number":1168,"context_line":"                last_modified \u003d datetime.now()"},{"line_number":1169,"context_line":""},{"line_number":1170,"context_line":"            last_modified_formatted \u003d last_modified.strftime("},{"line_number":1171,"context_line":"                \u0027%Y-%m-%dT%H:%M:%S.%f\u0027"},{"line_number":1172,"context_line":"            )"},{"line_number":1173,"context_line":"            seg_data \u003d {"},{"line_number":1174,"context_line":"                \u0027name\u0027: \u0027/\u0027 + seg_dict[\u0027path\u0027].lstrip(\u0027/\u0027),"},{"line_number":1175,"context_line":"                \u0027bytes\u0027: head_seg_resp.content_length,"},{"line_number":1176,"context_line":"                \u0027hash\u0027: head_seg_resp.etag,"},{"line_number":1177,"context_line":"                \u0027content_type\u0027: head_seg_resp.content_type,"},{"line_number":1178,"context_line":"                \u0027last_modified\u0027: last_modified_formatted"},{"line_number":1179,"context_line":"            }"},{"line_number":1180,"context_line":"            if seg_dict.get(\u0027range\u0027):"},{"line_number":1181,"context_line":"                seg_data[\u0027range\u0027] \u003d seg_dict[\u0027range\u0027]"},{"line_number":1182,"context_line":"            if config_true_value("}],"source_content_type":"text/x-python","patch_set":36,"id":"3f4b6375_033dcfc2","line":1179,"range":{"start_line":1170,"start_character":2,"end_line":1179,"end_character":13},"updated":"2017-11-08 07:31:43.000000000","message":"Same as above.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"52838d5f8799354a2b194132b1c5f8291d66a9c9","unresolved":false,"context_lines":[{"line_number":1078,"context_line":"            req.path)"},{"line_number":1079,"context_line":"        problem_segments \u003d []"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        too_many_object_segments \u003d len("},{"line_number":1082,"context_line":"            [1 for seg in parsed_data if \u0027path\u0027 in seg]"},{"line_number":1083,"context_line":"        ) \u003e self.max_manifest_segments"},{"line_number":1084,"context_line":"        if too_many_object_segments:"}],"source_content_type":"text/x-python","patch_set":37,"id":"df87a7cf_a5f2ca6e","line":1081,"updated":"2017-12-11 19:57:27.000000000","message":"If we\u0027re going to the trouble of actually building up an array (instead of just summing over an iterator), I\u0027d prefer something like\n\n object_segments \u003d [seg for seg in parsed_data if \u0027path\u0027 in seg]\n if len(object_segments) \u003e self.max_manifest_segments:\n     ...","commit_id":"d04bcb8c22916239d7aace9c51c010dc8b1214f0"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"58fa8e599a461407c6e369c399eae64acd33428b","unresolved":false,"context_lines":[{"line_number":1078,"context_line":"            req.path)"},{"line_number":1079,"context_line":"        problem_segments \u003d []"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        too_many_object_segments \u003d len("},{"line_number":1082,"context_line":"            [1 for seg in parsed_data if \u0027path\u0027 in seg]"},{"line_number":1083,"context_line":"        ) \u003e self.max_manifest_segments"},{"line_number":1084,"context_line":"        if too_many_object_segments:"}],"source_content_type":"text/x-python","patch_set":37,"id":"df87a7cf_90c32b38","line":1081,"in_reply_to":"df87a7cf_a5f2ca6e","updated":"2017-12-13 19:41:29.000000000","message":"agreed","commit_id":"d04bcb8c22916239d7aace9c51c010dc8b1214f0"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":348,"context_line":""},{"line_number":349,"context_line":"SLO_KEYS \u003d {"},{"line_number":350,"context_line":"    # required: optional"},{"line_number":351,"context_line":"    \u0027data\u0027: set(),"},{"line_number":352,"context_line":"    \u0027path\u0027: {\u0027range\u0027, \u0027etag\u0027, \u0027size_bytes\u0027},"},{"line_number":353,"context_line":"}"},{"line_number":354,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_94587a8a","line":351,"updated":"2017-12-18 07:39:33.000000000","message":"It might be yagni thing but is it worth to add \u0027etag\u0027 to optional field on \u0027data\u0027 as well as \u0027path\u0027?\n\nIf we could have the key here, we could assert the sent data is NOT corrupted via the etag check.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":348,"context_line":""},{"line_number":349,"context_line":"SLO_KEYS \u003d {"},{"line_number":350,"context_line":"    # required: optional"},{"line_number":351,"context_line":"    \u0027data\u0027: set(),"},{"line_number":352,"context_line":"    \u0027path\u0027: {\u0027range\u0027, \u0027etag\u0027, \u0027size_bytes\u0027},"},{"line_number":353,"context_line":"}"},{"line_number":354,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9f91af0f_90283d97","line":351,"in_reply_to":"bf8cb3f7_80a0aba7","updated":"2018-01-10 11:29:53.000000000","message":"ok. the etag at request header could be enough to validate the value. Make sense.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf9c5a17763a8402a2997a42ca6f6691ff86265d","unresolved":false,"context_lines":[{"line_number":348,"context_line":""},{"line_number":349,"context_line":"SLO_KEYS \u003d {"},{"line_number":350,"context_line":"    # required: optional"},{"line_number":351,"context_line":"    \u0027data\u0027: set(),"},{"line_number":352,"context_line":"    \u0027path\u0027: {\u0027range\u0027, \u0027etag\u0027, \u0027size_bytes\u0027},"},{"line_number":353,"context_line":"}"},{"line_number":354,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"bf8cb3f7_80a0aba7","line":351,"in_reply_to":"df87a7cf_94587a8a","updated":"2017-12-20 19:57:04.000000000","message":"Maybe? I feel like odds would be pretty good that a corruption would cause the body to no longer be valid JSON, producing an error similar to\n\n\u003e\u003e\u003e json.loads(\u0027[\"fo\\xff\"]\u0027)\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\n  File \"/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py\", line 338, in loads\n    return _default_decoder.decode(s)\n  File \"/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py\", line 366, in decode\n    obj, end \u003d self.raw_decode(s, idx\u003d_w(s, 0).end())\n  File \"/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py\", line 382, in raw_decode\n    obj, end \u003d self.scan_once(s, idx)\nUnicodeDecodeError: \u0027utf8\u0027 codec can\u0027t decode byte 0xff in position 2: invalid start byte\n\nBesides, if you\u0027re *really* worried about it, you can always include an ETag header which should be faithfully compared down at https://review.openstack.org/#/c/365371/39/swift/common/middleware/slo.py@1251\n\nPutting off for now; could always be an added feature later. I\u0027ve already got plans for another optional key...","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":401,"context_line":"            errors.append(\"Index %d: not a JSON object\" % seg_index)"},{"line_number":402,"context_line":"            continue"},{"line_number":403,"context_line":""},{"line_number":404,"context_line":"        for required in SLO_KEYS:"},{"line_number":405,"context_line":"            if required in seg_dict:"},{"line_number":406,"context_line":"                segment_type \u003d required"},{"line_number":407,"context_line":"                break"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_e0890526","line":404,"range":{"start_line":404,"start_character":13,"end_line":404,"end_character":20},"updated":"2017-12-18 07:39:33.000000000","message":"why not assigning segment_type directly?","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":416,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"},{"line_number":417,"context_line":"        if extraneous_keys:"},{"line_number":418,"context_line":"            errors.append("},{"line_number":419,"context_line":"                \"Index %d: extraneous keys %s\""},{"line_number":420,"context_line":"                % (seg_index,"},{"line_number":421,"context_line":"                   \", \".join(\u0027\"%s\"\u0027 % (ek,)"},{"line_number":422,"context_line":"                             for ek in sorted(extraneous_keys))))"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_5189a4c7","line":419,"updated":"2017-12-18 07:39:33.000000000","message":"I think we should have those extraneous keys tests against to data dict too, even it doesn\u0027t have any optional keys.\n\nI checked a simple case like as:\n\ndiff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py\nindex 230a1eb..42a0298 100644\n--- a/test/unit/common/middleware/test_slo.py\n+++ b/test/unit/common/middleware/test_slo.py\n@@ -730,6 +730,23 @@ class TestSloPutManifest(SloTestCase):\n         do_test(\u0027\u0027)\n         do_test(\u0027\u003d\u003d\u003d\u003d\u0027)\n \n+    def test_handle_invalid_data_dict(self):\n+        def do_test():\n+            postamble \u003d base64.b64encode(\u0027postamble\u0027)\n+            test_json_data \u003d json.dumps([{\u0027path\u0027: \u0027/cont/object\u0027,\n+                                          \u0027etag\u0027: \u0027etagoftheobjectsegment\u0027,\n+                                          \u0027size_bytes\u0027: 100},\n+                                         # path and data shouldn\u0027t exist\n+                                         # together\n+                                         {\u0027path\u0027: \u0027/cont/object\u0027,\n+                                          \u0027data\u0027: postamble}])\n+            req \u003d Request.blank(\u0027/v1/a/c/o\u0027, body\u003dtest_json_data)\n+            with self.assertRaises(HTTPException) as catcher:\n+                self.slo.handle_multipart_put(req, fake_start_response)\n+            self.assertEqual(catcher.exception.status_int, 400)\n+\n+        do_test()  # insufficient padding\n+\n     def test_handle_multipart_put_success_unicode(self):\n         test_json_data \u003d json.dumps([{\u0027path\u0027: u\u0027/cont/object\\u2661\u0027,\n                                       \u0027etag\u0027: \u0027etagoftheobjectsegment\u0027,\n\n\nIt passed (slo middleware raise 400 Bad Request) but the message is from here \u0027Index 1: extraneous keys \"data\"\u0027. I\u0027m worried about the key order in SLO_KEYS is environment-dependent and the message would not be directly intuitive.\n\nI suppose, the first limitation is we must use either \u0027data\u0027 or \u0027path\u0027 (not BOTH) and then, as the second limitation we cannot add extraneous keys other than the optional keys. Possibly we could check them and could make different message to notify \u0027what\u0027s wrong\u0027 for the client.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":481,"context_line":"                    errors.append(\"Index %d: unsatisfiable range\" % seg_index)"},{"line_number":482,"context_line":"                    continue"},{"line_number":483,"context_line":""},{"line_number":484,"context_line":"        if segment_type \u003d\u003d \u0027data\u0027:"},{"line_number":485,"context_line":"            # Validate that the supplied data is non-empty and base64-encoded"},{"line_number":486,"context_line":"            try:"},{"line_number":487,"context_line":"                data \u003d base64decode(seg_dict[\u0027data\u0027])"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_31eae0d7","line":484,"range":{"start_line":484,"start_character":7,"end_line":484,"end_character":10},"updated":"2017-12-18 07:39:33.000000000","message":"i like to change this if as `elif` to be strict as the other case.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf9c5a17763a8402a2997a42ca6f6691ff86265d","unresolved":false,"context_lines":[{"line_number":481,"context_line":"                    errors.append(\"Index %d: unsatisfiable range\" % seg_index)"},{"line_number":482,"context_line":"                    continue"},{"line_number":483,"context_line":""},{"line_number":484,"context_line":"        if segment_type \u003d\u003d \u0027data\u0027:"},{"line_number":485,"context_line":"            # Validate that the supplied data is non-empty and base64-encoded"},{"line_number":486,"context_line":"            try:"},{"line_number":487,"context_line":"                data \u003d base64decode(seg_dict[\u0027data\u0027])"}],"source_content_type":"text/x-python","patch_set":39,"id":"bf8cb3f7_80b74bd9","line":484,"range":{"start_line":484,"start_character":7,"end_line":484,"end_character":10},"in_reply_to":"df87a7cf_31eae0d7","updated":"2017-12-20 19:57:04.000000000","message":"Done","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dd0d5777a97cec0d0d438d79aae1fc6315b0fdd3","unresolved":false,"context_lines":[{"line_number":615,"context_line":"        last_sub_path \u003d None"},{"line_number":616,"context_line":"        for seg_dict in segments:"},{"line_number":617,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":618,"context_line":"                seg_dict[\u0027raw_data\u0027] \u003d base64decode(seg_dict.pop(\u0027data\u0027))"},{"line_number":619,"context_line":""},{"line_number":620,"context_line":"            seg_length \u003d self._segment_length(seg_dict)"},{"line_number":621,"context_line":"            if first_byte \u003e\u003d seg_length:"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_1794c8e7","line":618,"updated":"2017-12-15 00:06:00.000000000","message":"I\u0027m pretty sure this is a net-positive, but it\u0027s probably worth noting that we now have three different manifest formats:\n\n- the format the client sends,\n- the format we write on disk, and\n- the format that\u0027s in the proxy\u0027s head while processing GET requests.\n\nIt used to be that 2 \u0026 3 were the same, but not any more... worst-case, we can merge them again later as a follow-up without any upgrade troubles.\n\nShould we have a friendlier error here if a manifest somehow slips in that has bad data? Probably at least worth logging a warning.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dd0d5777a97cec0d0d438d79aae1fc6315b0fdd3","unresolved":false,"context_lines":[{"line_number":884,"context_line":"                response_headers.append((header, value))"},{"line_number":885,"context_line":""},{"line_number":886,"context_line":"        # Decode any inlined data, and calculate content_length \u0026 etag"},{"line_number":887,"context_line":"        # if necessary"},{"line_number":888,"context_line":"        if slo_etag is None:"},{"line_number":889,"context_line":"            calculated_etag \u003d md5()"},{"line_number":890,"context_line":"        if content_length is None:"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_d3d4144a","line":887,"updated":"2017-12-15 00:06:00.000000000","message":"Probably should move this comment down a bit.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf9c5a17763a8402a2997a42ca6f6691ff86265d","unresolved":false,"context_lines":[{"line_number":884,"context_line":"                response_headers.append((header, value))"},{"line_number":885,"context_line":""},{"line_number":886,"context_line":"        # Decode any inlined data, and calculate content_length \u0026 etag"},{"line_number":887,"context_line":"        # if necessary"},{"line_number":888,"context_line":"        if slo_etag is None:"},{"line_number":889,"context_line":"            calculated_etag \u003d md5()"},{"line_number":890,"context_line":"        if content_length is None:"}],"source_content_type":"text/x-python","patch_set":39,"id":"bf8cb3f7_40e623eb","line":887,"in_reply_to":"df87a7cf_d3d4144a","updated":"2017-12-20 19:57:04.000000000","message":"Done","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b3a0342434331ea14e080f18032bbd3c0fef6144","unresolved":false,"context_lines":[{"line_number":961,"context_line":"            limit_after\u003dself.slo.rate_limit_after_segment,"},{"line_number":962,"context_line":"            ratelimit_if\u003dratelimit_predicate)"},{"line_number":963,"context_line":""},{"line_number":964,"context_line":"        # self._segment_listing_iterator gives us 3-tuples of (segment dict,"},{"line_number":965,"context_line":"        # start byte, end byte), but SegmentedIterable wants (obj path, etag,"},{"line_number":966,"context_line":"        # size, start byte, end byte), so we clean that up here"},{"line_number":967,"context_line":"        segment_listing_iter \u003d ("},{"line_number":968,"context_line":"            seg_dict if \u0027raw_data\u0027 in seg_dict else"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_b147d077","line":965,"range":{"start_line":964,"start_character":50,"end_line":965,"end_character":32},"updated":"2017-12-18 07:39:33.000000000","message":"This is already wrong. This patch changes the return value from tuple to dict.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf9c5a17763a8402a2997a42ca6f6691ff86265d","unresolved":false,"context_lines":[{"line_number":961,"context_line":"            limit_after\u003dself.slo.rate_limit_after_segment,"},{"line_number":962,"context_line":"            ratelimit_if\u003dratelimit_predicate)"},{"line_number":963,"context_line":""},{"line_number":964,"context_line":"        # self._segment_listing_iterator gives us 3-tuples of (segment dict,"},{"line_number":965,"context_line":"        # start byte, end byte), but SegmentedIterable wants (obj path, etag,"},{"line_number":966,"context_line":"        # size, start byte, end byte), so we clean that up here"},{"line_number":967,"context_line":"        segment_listing_iter \u003d ("},{"line_number":968,"context_line":"            seg_dict if \u0027raw_data\u0027 in seg_dict else"}],"source_content_type":"text/x-python","patch_set":39,"id":"bf8cb3f7_c0d9d322","line":965,"range":{"start_line":964,"start_character":50,"end_line":965,"end_character":32},"in_reply_to":"df87a7cf_b147d077","updated":"2017-12-20 19:57:04.000000000","message":"Done","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":209,"context_line":"-------------------------"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"When uploading a manifest, users can include \u0027data\u0027 segments that should"},{"line_number":212,"context_line":"be included between objects. The data in these segments must be"},{"line_number":213,"context_line":"base64-encoded binary data and will be included in the etag of the"},{"line_number":214,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":215,"context_line":"referenced as separate objects."}],"source_content_type":"text/x-python","patch_set":41,"id":"9f91af0f_c953806a","line":212,"range":{"start_line":212,"start_character":4,"end_line":212,"end_character":28},"updated":"2018-01-10 11:29:53.000000000","message":"nits) actually an inline data can be placed as the first segment.","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":209,"context_line":"-------------------------"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"When uploading a manifest, users can include \u0027data\u0027 segments that should"},{"line_number":212,"context_line":"be included between objects. The data in these segments must be"},{"line_number":213,"context_line":"base64-encoded binary data and will be included in the etag of the"},{"line_number":214,"context_line":"resulting large object exactly as if that data had been uploaded and"},{"line_number":215,"context_line":"referenced as separate objects."}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_cc8f37cb","line":212,"range":{"start_line":212,"start_character":4,"end_line":212,"end_character":28},"in_reply_to":"9f91af0f_c953806a","updated":"2018-01-17 00:48:33.000000000","message":"Done","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    This feature is primarily aimed at reducing the need for storing"},{"line_number":220,"context_line":"    many tiny objects, and as such any supplied data must fit within"},{"line_number":221,"context_line":"    the maximum manifest size (default is 8MiB). This maximum size"},{"line_number":222,"context_line":"    can be configured via ``max_manifest_size`` in proxy-server.conf."},{"line_number":223,"context_line":""},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"-------------------------"}],"source_content_type":"text/x-python","patch_set":41,"id":"9f91af0f_89ded8d7","line":222,"updated":"2018-01-10 11:29:53.000000000","message":"Thinking of the purpose, perhaps, we may have a limitation of the max size for each inline data. e.g. 1MB at max.","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    This feature is primarily aimed at reducing the need for storing"},{"line_number":220,"context_line":"    many tiny objects, and as such any supplied data must fit within"},{"line_number":221,"context_line":"    the maximum manifest size (default is 8MiB). This maximum size"},{"line_number":222,"context_line":"    can be configured via ``max_manifest_size`` in proxy-server.conf."},{"line_number":223,"context_line":""},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"-------------------------"}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_c64cf6c9","line":222,"in_reply_to":"9f91af0f_89ded8d7","updated":"2018-01-17 00:48:33.000000000","message":"Meh -- such a restriction would be trivial to get around as-is: include multiple consecutive data segments.\n\nMore broadly, though, I don\u0027t see much point in limiting it per-segment -- the important things are:\n\nPrevent the proxy server from using too much memory by\n\n* limiting how big a single manifest can be\n* limiting how many manifests can be nested at a time\n\nPrevent a client from holding a connection for too long by\n\n- limiting max_get_time\n\nPrevent downloads from hammering your object servers by\n\n- using the rate_limit_* options\n\nThe only argument I can see for limiting data segment size is to limit memory consumption, and that\u0027s already addressed by limiting the overall manifest size.\n\nReally, I\u0027m not even sure that we need or ought to limit the number of object-backed segments per manifest... I think that *was* good and useful when we needed to worry about clients timing out on PUT, but now that you can send\n\n ?heartbeat\u003don\n\nI really want to bump that up to at least 10k...","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":342,"context_line":""},{"line_number":343,"context_line":"DEFAULT_RATE_LIMIT_UNDER_SIZE \u003d 1024 ** 2  # 1 MiB"},{"line_number":344,"context_line":"DEFAULT_MAX_MANIFEST_SEGMENTS \u003d 1000"},{"line_number":345,"context_line":"DEFAULT_MAX_MANIFEST_SIZE \u003d 8 * (1024 ** 2)  # 8 MiB"},{"line_number":346,"context_line":"DEFAULT_YIELD_FREQUENCY \u003d 10"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":""}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_a6c44af6","line":345,"updated":"2018-01-17 00:48:33.000000000","message":"So a proxy might need 80MB to service a single GET, if the SLO is deeply-nested... but we\u0027d be hitting 20MB for a single request previously... what\u0027s the worst, right?","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                              % (seg_index,))"},{"line_number":496,"context_line":"                continue"},{"line_number":497,"context_line":"            # re-encode to strip excess padding"},{"line_number":498,"context_line":"            seg_dict[\u0027data\u0027] \u003d base64.b64encode(data)"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"    if parsed_data and all(\u0027data\u0027 in d for d in parsed_data):"},{"line_number":501,"context_line":"        errors.append(\"Inline data segments require at least one \""}],"source_content_type":"text/x-python","patch_set":41,"id":"9f91af0f_bace7860","line":498,"updated":"2018-01-10 11:29:53.000000000","message":"Curious. I thought this *data* is not from stripped value so it doesn\u0027t seem to work. However, FWIW, it seems like b64decode strips the padding \"\u003d\" at the b64decode so it\u0027s realizing this seems to work well as the comment.","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                              % (seg_index,))"},{"line_number":496,"context_line":"                continue"},{"line_number":497,"context_line":"            # re-encode to strip excess padding"},{"line_number":498,"context_line":"            seg_dict[\u0027data\u0027] \u003d base64.b64encode(data)"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"    if parsed_data and all(\u0027data\u0027 in d for d in parsed_data):"},{"line_number":501,"context_line":"        errors.append(\"Inline data segments require at least one \""}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_c67dd6b2","line":498,"in_reply_to":"9f91af0f_bace7860","updated":"2018-01-17 00:48:33.000000000","message":"Better if I re-word as\n\n # re-encode to normalize padding\n\n?","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":893,"context_line":"            # Decode any inlined data; it\u0027s important that we do this *before*"},{"line_number":894,"context_line":"            # calculating the segment length and etag"},{"line_number":895,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":896,"context_line":"                seg_dict[\u0027raw_data\u0027] \u003d base64.b64decode(seg_dict.pop(\u0027data\u0027))"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":"            if slo_etag is None:"},{"line_number":899,"context_line":"                if \u0027raw_data\u0027 in seg_dict:"}],"source_content_type":"text/x-python","patch_set":41,"id":"9f91af0f_9a285497","line":896,"range":{"start_line":896,"start_character":39,"end_line":896,"end_character":77},"updated":"2018-01-10 11:29:53.000000000","message":"ok. this should be \"re-encoded\" value at the PUT manifest.","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":893,"context_line":"            # Decode any inlined data; it\u0027s important that we do this *before*"},{"line_number":894,"context_line":"            # calculating the segment length and etag"},{"line_number":895,"context_line":"            if \u0027data\u0027 in seg_dict:"},{"line_number":896,"context_line":"                seg_dict[\u0027raw_data\u0027] \u003d base64.b64decode(seg_dict.pop(\u0027data\u0027))"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":"            if slo_etag is None:"},{"line_number":899,"context_line":"                if \u0027raw_data\u0027 in seg_dict:"}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_a6376aff","line":896,"range":{"start_line":896,"start_character":39,"end_line":896,"end_character":77},"in_reply_to":"9f91af0f_9a285497","updated":"2018-01-17 00:48:33.000000000","message":"Correct. So there shouldn\u0027t be any errors at this point (unless you went and used a direct client to store a manifest without going through slo or something, but in that case we are well outside \"normal behavior\").","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3808ff07fc27e3653f795cca98d386e22d367c7b","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2016 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":42,"id":"7f96bb07_5a6fa4fb","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":20},"updated":"2018-01-30 10:49:13.000000000","message":"it\u0027s 2018 now.. I hope this isn\u0027t an indication on how long you\u0027ve been working on this patch.","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d5afbfd7dbbc32014b658545d414be298a769960","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Copyright (c) 2016 OpenStack Foundation"},{"line_number":2,"context_line":"#"},{"line_number":3,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"},{"line_number":4,"context_line":"# you may not use this file except in compliance with the License."}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_3d013406","line":1,"range":{"start_line":1,"start_character":16,"end_line":1,"end_character":20},"in_reply_to":"7f96bb07_5a6fa4fb","updated":"2018-01-30 21:42:00.000000000","message":"It totally is -- Joel started this in September 2016!","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3808ff07fc27e3653f795cca98d386e22d367c7b","unresolved":false,"context_lines":[{"line_number":413,"context_line":"            continue"},{"line_number":414,"context_line":""},{"line_number":415,"context_line":"        allowed_keys \u003d SLO_KEYS[segment_type].union([segment_type])"},{"line_number":416,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"},{"line_number":417,"context_line":"        if extraneous_keys:"},{"line_number":418,"context_line":"            errors.append("},{"line_number":419,"context_line":"                \"Index %d: extraneous keys %s\""}],"source_content_type":"text/x-python","patch_set":42,"id":"7f96bb07_d5fce5b6","line":416,"updated":"2018-01-30 10:49:13.000000000","message":"seeing as allowed_keys is already a set we could just:\n\n  extraneous_keys \u003d set(seg_dict.keys()).difference(allowed_keys)\n\nor something, but meh 1/2 a dozen of one 6 of another :)","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d5afbfd7dbbc32014b658545d414be298a769960","unresolved":false,"context_lines":[{"line_number":413,"context_line":"            continue"},{"line_number":414,"context_line":""},{"line_number":415,"context_line":"        allowed_keys \u003d SLO_KEYS[segment_type].union([segment_type])"},{"line_number":416,"context_line":"        extraneous_keys \u003d [k for k in seg_dict if k not in allowed_keys]"},{"line_number":417,"context_line":"        if extraneous_keys:"},{"line_number":418,"context_line":"            errors.append("},{"line_number":419,"context_line":"                \"Index %d: extraneous keys %s\""}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_1d7318b3","line":416,"in_reply_to":"7f96bb07_d5fce5b6","updated":"2018-01-30 21:42:00.000000000","message":"I think I\u0027d debated about that at some point, then came back around when thinking about how it would\n\n- iterate through all keys to build up the initial set, then\n- iterate through them all again to calculate the difference\n\ninstead of just iterating through them all once.","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"}],"swift/common/request_helpers.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d3e30614a5aff8fedc2fb12c948f476bacbc7198","unresolved":false,"context_lines":[{"line_number":306,"context_line":"    \"\"\""},{"line_number":307,"context_line":"    Iterable that handles the older style 5-tuple iterables previously passed"},{"line_number":308,"context_line":"    to SegmentedIterable. SegmentedIterable has now been updated to accept an"},{"line_number":309,"context_line":"    iterable over dicts to allow extensibility of SLO manifests."},{"line_number":310,"context_line":"    \"\"\""},{"line_number":311,"context_line":"    def __init__(self, iterable):"},{"line_number":312,"context_line":"        self.iterable \u003d iterable"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_72909896","line":309,"updated":"2016-10-27 07:06:40.000000000","message":"Can we just convert all of the callers to use the new style?","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"29be545c47597e03ef38dc904518d7f7dd34a791","unresolved":false,"context_lines":[{"line_number":306,"context_line":"    \"\"\""},{"line_number":307,"context_line":"    Iterable that handles the older style 5-tuple iterables previously passed"},{"line_number":308,"context_line":"    to SegmentedIterable. SegmentedIterable has now been updated to accept an"},{"line_number":309,"context_line":"    iterable over dicts to allow extensibility of SLO manifests."},{"line_number":310,"context_line":"    \"\"\""},{"line_number":311,"context_line":"    def __init__(self, iterable):"},{"line_number":312,"context_line":"        self.iterable \u003d iterable"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_fb48e45d","line":309,"in_reply_to":"ba5da102_72909896","updated":"2016-10-27 14:04:51.000000000","message":"I can certainly convert SLO and DLO, but I can\u0027t guarantee that no external libraries are relying on the old style. Everything in the swift repo should certainly be migrated.","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"6e29bf88be90157d6665a03d2344b139cb886d30","unresolved":false,"context_lines":[{"line_number":306,"context_line":"    \"\"\""},{"line_number":307,"context_line":"    Iterable that handles the older style 5-tuple iterables previously passed"},{"line_number":308,"context_line":"    to SegmentedIterable. SegmentedIterable has now been updated to accept an"},{"line_number":309,"context_line":"    iterable over dicts to allow extensibility of SLO manifests."},{"line_number":310,"context_line":"    \"\"\""},{"line_number":311,"context_line":"    def __init__(self, iterable):"},{"line_number":312,"context_line":"        self.iterable \u003d iterable"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_3bab2d1f","line":309,"in_reply_to":"ba5da102_fb48e45d","updated":"2016-10-28 08:14:57.000000000","message":"Speaking as someone who has an external-to-Swift consumer of SegmentedIterable, while I appreciate the effort, I\u0027d rather keep this code out of Swift and just let my middleware have the cruft in it. I\u0027m hopeful that I could get rid of my own iterator adapter someday, but this one in Swift would live forever.","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d3e30614a5aff8fedc2fb12c948f476bacbc7198","unresolved":false,"context_lines":[{"line_number":670,"context_line":"                continue"},{"line_number":671,"context_line":""},{"line_number":672,"context_line":"            # Handle any bytes required from the postamble"},{"line_number":673,"context_line":"            if first_byte \u003e (preamble_size + seg_size):"},{"line_number":674,"context_line":"                first_postamble_byte \u003d first_byte - ("},{"line_number":675,"context_line":"                    preamble_size + seg_size"},{"line_number":676,"context_line":"                )"}],"source_content_type":"text/x-python","patch_set":2,"id":"ba5da102_273fc426","line":673,"range":{"start_line":673,"start_character":45,"end_line":673,"end_character":53},"updated":"2016-10-27 07:06:40.000000000","message":"This might be None, which is causing the 500s in the func tests.","commit_id":"809321c1a6a585fcca11a3f96331880c7f53862b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":310,"context_line":"    :param app: WSGI application from which segments will come"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    :param listing_iter: iterable yielding the object segments to fetch,"},{"line_number":313,"context_line":"        along with the byte subranges to fetch, in the form of a 5-tuple"},{"line_number":314,"context_line":"        (object-path, object-etag, object-size, first-byte, last-byte)."},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"        If object-etag is None, no MD5 verification will be done."}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_6c94b742","line":313,"range":{"start_line":313,"start_character":48,"end_line":313,"end_character":72},"updated":"2016-12-27 20:28:58.000000000","message":"Comment needs updating.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":310,"context_line":"    :param app: WSGI application from which segments will come"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    :param listing_iter: iterable yielding the object segments to fetch,"},{"line_number":313,"context_line":"        along with the byte subranges to fetch, in the form of a 5-tuple"},{"line_number":314,"context_line":"        (object-path, object-etag, object-size, first-byte, last-byte)."},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"        If object-etag is None, no MD5 verification will be done."}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_20c93979","line":313,"range":{"start_line":313,"start_character":48,"end_line":313,"end_character":72},"in_reply_to":"da4df55a_6c94b742","updated":"2017-01-13 23:35:22.000000000","message":"Done","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":395,"context_line":""},{"line_number":396,"context_line":"                # We can only coalesce if"},{"line_number":397,"context_line":"                #  - there is no segment preamble"},{"line_number":398,"context_line":"                #  - there is no segment postamble"},{"line_number":399,"context_line":"                #  - we have a previous request and the paths match"},{"line_number":400,"context_line":"                #  - we know the segment size (so we can validate the ranges)"},{"line_number":401,"context_line":"                can_coalesce \u003d ("}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_0c3fab87","line":398,"updated":"2016-12-27 20:28:58.000000000","message":"Ugh, yeah, it\u0027d be pretty terrible trying to inject the pre-/postambles mid-stream. I wager it\u0027s worth it to make extra requests in the unlikely event that you\u0027ve multiple segments pointing to the same object, each with pre-/postambles.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fb3972fdde6379cfd8a906db5d7d6c7451b53599","unresolved":false,"context_lines":[{"line_number":354,"context_line":"        self.validated_first_segment \u003d False"},{"line_number":355,"context_line":"        self.current_resp \u003d None"},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"    def _coalescing_iter(self):"},{"line_number":358,"context_line":"        pending_req \u003d None"},{"line_number":359,"context_line":"        pending_dict \u003d None"},{"line_number":360,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"7ffa3b31_6ad372d6","line":357,"updated":"2017-04-21 01:41:09.000000000","message":"Probably a good rename... So many iters now, though!","commit_id":"ebc637c450b56cccbd8fb010b4c1498bae88ddef"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"52838d5f8799354a2b194132b1c5f8291d66a9c9","unresolved":false,"context_lines":[{"line_number":430,"context_line":"        try:"},{"line_number":431,"context_line":"            for seg_req, seg_etag, seg_size in self._coalesce_requests():"},{"line_number":432,"context_line":"                if isinstance(seg_req, bytes):"},{"line_number":433,"context_line":"                    chunk \u003d seg_req  # ugly, awful overloading"},{"line_number":434,"context_line":"                    if bytes_left is None:"},{"line_number":435,"context_line":"                        yield chunk"},{"line_number":436,"context_line":"                    elif bytes_left \u003e\u003d len(chunk):"}],"source_content_type":"text/x-python","patch_set":37,"id":"df87a7cf_eb574125","line":433,"updated":"2017-12-11 19:57:27.000000000","message":"Well, I wasn\u0027t lying... it\u0027s pretty awful. With a partially-data-backed slo nested inside a \"normal\" slo:\n\n  ...\n  File \"/vagrant/swift/swift/common/middleware/slo.py\", line 978, in _manifest_get_response\n    segmented_iter.validate_first_segment()\n  File \"/vagrant/swift/swift/common/request_helpers.py\", line 561, in validate_first_segment\n    self.peeked_chunk \u003d next(self.app_iter)\n  File \"/vagrant/swift/swift/common/request_helpers.py\", line 442, in _internal_iter\n    seg_resp \u003d seg_req.get_response(self.app)\nAttributeError: \u0027unicode\u0027 object has no attribute \u0027get_response\u0027 (txn: txe4e38b25fa6c43b78dd16-005a2ee1d8)","commit_id":"d04bcb8c22916239d7aace9c51c010dc8b1214f0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dd0d5777a97cec0d0d438d79aae1fc6315b0fdd3","unresolved":false,"context_lines":[{"line_number":430,"context_line":"        try:"},{"line_number":431,"context_line":"            for seg_req, seg_etag, seg_size in self._coalesce_requests():"},{"line_number":432,"context_line":"                if isinstance(seg_req, bytes):"},{"line_number":433,"context_line":"                    chunk \u003d seg_req  # ugly, awful overloading"},{"line_number":434,"context_line":"                    if bytes_left is None:"},{"line_number":435,"context_line":"                        yield chunk"},{"line_number":436,"context_line":"                    elif bytes_left \u003e\u003d len(chunk):"}],"source_content_type":"text/x-python","patch_set":39,"id":"df87a7cf_b35f10e4","line":433,"updated":"2017-12-15 00:06:00.000000000","message":"I\u0027m still not entirely happy about this, but I don\u0027t have any better idea, either.","commit_id":"b92f79aacd7bdb4dbf567bdb4a50e98c365a3ada"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"c400c081ac0a2649c7b456e7ebac5a3e0497db13","unresolved":false,"context_lines":[{"line_number":455,"context_line":""},{"line_number":456,"context_line":"        try:"},{"line_number":457,"context_line":"            for seg_req, seg_etag, seg_size in self._coalesce_requests():"},{"line_number":458,"context_line":"                if isinstance(seg_req, bytes):"},{"line_number":459,"context_line":"                    chunk \u003d seg_req  # ugly, awful overloading"},{"line_number":460,"context_line":"                    if bytes_left is None:"},{"line_number":461,"context_line":"                        yield chunk"}],"source_content_type":"text/x-python","patch_set":41,"id":"9f91af0f_993f0186","line":458,"updated":"2018-01-10 11:29:53.000000000","message":"oops? ah... with this change, seg_req may be not requests anymore.\n\nAs Tim commented at the old patch set, I do not like it too but no idea for now to improve.","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1c64b3a416c60b6191410a22727eb131b9789c53","unresolved":false,"context_lines":[{"line_number":455,"context_line":""},{"line_number":456,"context_line":"        try:"},{"line_number":457,"context_line":"            for seg_req, seg_etag, seg_size in self._coalesce_requests():"},{"line_number":458,"context_line":"                if isinstance(seg_req, bytes):"},{"line_number":459,"context_line":"                    chunk \u003d seg_req  # ugly, awful overloading"},{"line_number":460,"context_line":"                    if bytes_left is None:"},{"line_number":461,"context_line":"                        yield chunk"}],"source_content_type":"text/x-python","patch_set":41,"id":"7f96bb07_ac9bcb15","line":458,"in_reply_to":"9f91af0f_993f0186","updated":"2018-01-17 00:48:33.000000000","message":"I might be able to get a slight improvement out of an extra var name...","commit_id":"734c8a18acf2fb0de1395e509cd8e643a35a4367"}],"test/functional/test_slo.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d8a7e3237f93f96d8c39de4134516d27e0049d3a","unresolved":false,"context_lines":[{"line_number":1061,"context_line":"        grouped_file_contents \u003d ["},{"line_number":1062,"context_line":"            (char, sum(1 for _char in grp))"},{"line_number":1063,"context_line":"            for char, grp in itertools.groupby(file_contents)]"},{"line_number":1064,"context_line":"        self.maxDiff\u003dNone"},{"line_number":1065,"context_line":"        self.assertEqual(["},{"line_number":1066,"context_line":"            (\u0027A\u0027, 1),"},{"line_number":1067,"context_line":"            (\u0027P\u0027, 1),"}],"source_content_type":"text/x-python","patch_set":31,"id":"3f4b6375_23289632","line":1064,"updated":"2017-10-19 17:28:58.000000000","message":"Whoops! Left-over from debugging.\n\nBut maybe it\u0027s actually kinda useful if there\u0027s a failure? Or I should switch to z/y/x/w/v for APRE, APOS, BPOS, CPRE, and CPOS","commit_id":"1969ad9a6c3035ee55a51535cb86c30c6dd2e361"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":207,"context_line":"                 \u0027size_bytes\u0027: None, \u0027range\u0027: \u0027-1048578\u0027},"},{"line_number":208,"context_line":"            ]), parms\u003d{\u0027multipart-manifest\u0027: \u0027put\u0027})"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"        file_item \u003d cls.container.file(\"pre-postamble-manifest\")"},{"line_number":211,"context_line":"        file_item.write("},{"line_number":212,"context_line":"            json.dumps(["},{"line_number":213,"context_line":"                {\u0027data\u0027: base64.b64encode(\u0027APRE\u0027 * 8)},"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_b4f7a559","line":210,"range":{"start_line":210,"start_character":40,"end_line":210,"end_character":62},"updated":"2017-11-16 16:44:23.000000000","message":"There are a lot of \u0027pre-postamble\u0027 references in this test file now. Maybe they should be renamed to \u0027mixed-data-object\u0027 \n(or something similar) to avoid confusion.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"}],"test/unit/common/middleware/test_slo.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[{"line_number":2109,"context_line":"        self.assertEqual(status, \u0027200 OK\u0027)"},{"line_number":2110,"context_line":"        self.assertEqual(headers[\u0027Content-Length\u0027], \u002732\u0027)"},{"line_number":2111,"context_line":"        self.assertEqual(headers[\u0027Content-Type\u0027], \u0027application/json\u0027)"},{"line_number":2112,"context_line":"        # self.assertEqual(body, \u0027aaaaaaaaccccccccbbbbbbbbdddddddd\u0027)"},{"line_number":2113,"context_line":""},{"line_number":2114,"context_line":"        self.assertEqual("},{"line_number":2115,"context_line":"            self.app.calls,"}],"source_content_type":"text/x-python","patch_set":13,"id":"da4df55a_574140a8","line":2112,"updated":"2016-12-27 20:28:58.000000000","message":"?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[{"line_number":2109,"context_line":"        self.assertEqual(status, \u0027200 OK\u0027)"},{"line_number":2110,"context_line":"        self.assertEqual(headers[\u0027Content-Length\u0027], \u002732\u0027)"},{"line_number":2111,"context_line":"        self.assertEqual(headers[\u0027Content-Type\u0027], \u0027application/json\u0027)"},{"line_number":2112,"context_line":"        # self.assertEqual(body, \u0027aaaaaaaaccccccccbbbbbbbbdddddddd\u0027)"},{"line_number":2113,"context_line":""},{"line_number":2114,"context_line":"        self.assertEqual("},{"line_number":2115,"context_line":"            self.app.calls,"}],"source_content_type":"text/x-python","patch_set":13,"id":"7a3c09a3_1dfd8a3a","line":2112,"in_reply_to":"da4df55a_574140a8","updated":"2017-01-13 23:35:22.000000000","message":"No idea, I\u0027m obviously mad!","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0adcadc41a32f82db124b216d304e17669476d2d","unresolved":false,"context_lines":[{"line_number":781,"context_line":"        self.assertTrue(headers[\u0027Content-Type\u0027].endswith(\u0027;swift_bytes\u003d7\u0027))"},{"line_number":782,"context_line":"        self.assertEqual(len(manifest_data), 1)"},{"line_number":783,"context_line":"        self.assertEqual(manifest_data[0][\u0027bytes\u0027], 0)"},{"line_number":784,"context_line":""},{"line_number":785,"context_line":"    def test_handle_unsatisfiable_ranges(self):"},{"line_number":786,"context_line":"        bad_data \u003d json.dumps("},{"line_number":787,"context_line":"            [{\u0027path\u0027: \u0027/checktest/a_1\u0027, \u0027etag\u0027: None,"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_01def510","line":784,"updated":"2017-03-01 22:48:25.000000000","message":"Seems like we need some invalid pre/postamble tests?","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"f99f10a532c53f18d14f3592f2e0849028db3ce6","unresolved":false,"context_lines":[{"line_number":781,"context_line":"        self.assertTrue(headers[\u0027Content-Type\u0027].endswith(\u0027;swift_bytes\u003d7\u0027))"},{"line_number":782,"context_line":"        self.assertEqual(len(manifest_data), 1)"},{"line_number":783,"context_line":"        self.assertEqual(manifest_data[0][\u0027bytes\u0027], 0)"},{"line_number":784,"context_line":""},{"line_number":785,"context_line":"    def test_handle_unsatisfiable_ranges(self):"},{"line_number":786,"context_line":"        bad_data \u003d json.dumps("},{"line_number":787,"context_line":"            [{\u0027path\u0027: \u0027/checktest/a_1\u0027, \u0027etag\u0027: None,"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba2be162_ad84b00a","line":784,"in_reply_to":"ba2be162_01def510","updated":"2017-03-05 11:54:07.000000000","message":"Good point, will fix this.","commit_id":"4b9d7f1ed1f510eead355fac42d6ef17b436f09c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4d9859283da5bbf715a7e6899cc745e72c7346da","unresolved":false,"context_lines":[{"line_number":2964,"context_line":"            7 * 3600,  # check after yielding first segment"},{"line_number":2965,"context_line":"            21 * 3600,  # second segment takes *even longer* (ostensibly"},{"line_number":2966,"context_line":"                        # something to do with submanifests)"},{"line_number":2967,"context_line":"            28 * 3600,  # check after yielding third segment -- timeout"},{"line_number":2968,"context_line":"        ]"},{"line_number":2969,"context_line":"        req \u003d Request.blank("},{"line_number":2970,"context_line":"            \u0027/v1/AUTH_test/gettest/manifest-abcd\u0027,"}],"source_content_type":"text/x-python","patch_set":30,"id":"5f4e5783_fa0e6c8b","line":2967,"updated":"2017-10-12 19:51:04.000000000","message":"Oh man, _time_limit_iter cleans up this story *so much*! Kudos!","commit_id":"a6e5370a2de6d3b01881082a21c56ae4f75e58bf"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"182b2d40676f7160a0a638aeac97d761cd9e2c6a","unresolved":false,"context_lines":[{"line_number":3305,"context_line":"        self.assertIn((\u0027Etag\u0027, \u0027\"%s\"\u0027 % slo_etag), headers)"},{"line_number":3306,"context_line":"        self.assertIn((\u0027Content-Length\u0027, \u002739\u0027), headers)"},{"line_number":3307,"context_line":""},{"line_number":3308,"context_line":"        # Test last byte first pre-amble to first byte of second postamble"},{"line_number":3309,"context_line":"        req \u003d Request.blank("},{"line_number":3310,"context_line":"            \u0027/v1/AUTH_test/gettest/manifest-multi-prepostamble\u0027,"},{"line_number":3311,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027},"}],"source_content_type":"text/x-python","patch_set":36,"id":"1f485f77_148af9bd","line":3308,"range":{"start_line":3308,"start_character":30,"end_line":3308,"end_character":41},"updated":"2017-11-16 16:44:23.000000000","message":"We probably need to update these comments to remove pre/post-amble references. I have no doubt this will cause confusion in the future.","commit_id":"7f66606f9fa61ed24c5438e54f58b102c1acc53b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d5afbfd7dbbc32014b658545d414be298a769960","unresolved":false,"context_lines":[{"line_number":2689,"context_line":"                 \u0027hash\u0027: md5hex(\u0027body%02d\u0027 % i),"},{"line_number":2690,"context_line":"                 \u0027bytes\u0027: \u00276\u0027,"},{"line_number":2691,"context_line":"                 \u0027content_type\u0027: \u0027text/plain\u0027},"},{"line_number":2692,"context_line":"                {\u0027name\u0027: \u0027/gettest/man%d\u0027 % (i + 1),"},{"line_number":2693,"context_line":"                 \u0027hash\u0027: \u0027man%d\u0027 % (i + 1),"},{"line_number":2694,"context_line":"                 \u0027sub_slo\u0027: True,"},{"line_number":2695,"context_line":"                 \u0027bytes\u0027: len(manifest_json),"}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_78eeeaed","line":2692,"updated":"2018-01-30 21:42:00.000000000","message":"I could repro Matt\u0027s issue by making similar changes here...","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d5afbfd7dbbc32014b658545d414be298a769960","unresolved":false,"context_lines":[{"line_number":2692,"context_line":"                {\u0027name\u0027: \u0027/gettest/man%d\u0027 % (i + 1),"},{"line_number":2693,"context_line":"                 \u0027hash\u0027: \u0027man%d\u0027 % (i + 1),"},{"line_number":2694,"context_line":"                 \u0027sub_slo\u0027: True,"},{"line_number":2695,"context_line":"                 \u0027bytes\u0027: len(manifest_json),"},{"line_number":2696,"context_line":"                 \u0027content_type\u0027: \u0027application/json\u0027}]"},{"line_number":2697,"context_line":""},{"line_number":2698,"context_line":"            manifest_json \u003d json.dumps(manifest_data)"}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_d8937e5a","line":2695,"updated":"2018-01-30 21:42:00.000000000","message":"... but if I crank this up to something like\n\n len(manifest_json) * 3\n\nit goes away. I\u0027m not sure that this set-up is actually representative of what we expect to have stored on disk...\n\nI mean, if we *weren\u0027t* exceeding the recursion depth, we\u0027d *definitely* be expecting warnings about short reads!","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"72f30747db22540b6dce02c5a8fe66f8256e8aa3","unresolved":false,"context_lines":[{"line_number":2692,"context_line":"                {\u0027name\u0027: \u0027/gettest/man%d\u0027 % (i + 1),"},{"line_number":2693,"context_line":"                 \u0027hash\u0027: \u0027man%d\u0027 % (i + 1),"},{"line_number":2694,"context_line":"                 \u0027sub_slo\u0027: True,"},{"line_number":2695,"context_line":"                 \u0027bytes\u0027: len(manifest_json),"},{"line_number":2696,"context_line":"                 \u0027content_type\u0027: \u0027application/json\u0027}]"},{"line_number":2697,"context_line":""},{"line_number":2698,"context_line":"            manifest_json \u003d json.dumps(manifest_data)"}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_147c4e1d","line":2695,"in_reply_to":"5f93b717_5838ceed","updated":"2018-01-31 02:15:03.000000000","message":"Done","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"6019e82b8f553e9c3546a2d590890db8e1f6aa0e","unresolved":false,"context_lines":[{"line_number":2692,"context_line":"                {\u0027name\u0027: \u0027/gettest/man%d\u0027 % (i + 1),"},{"line_number":2693,"context_line":"                 \u0027hash\u0027: \u0027man%d\u0027 % (i + 1),"},{"line_number":2694,"context_line":"                 \u0027sub_slo\u0027: True,"},{"line_number":2695,"context_line":"                 \u0027bytes\u0027: len(manifest_json),"},{"line_number":2696,"context_line":"                 \u0027content_type\u0027: \u0027application/json\u0027}]"},{"line_number":2697,"context_line":""},{"line_number":2698,"context_line":"            manifest_json \u003d json.dumps(manifest_data)"}],"source_content_type":"text/x-python","patch_set":42,"id":"5f93b717_5838ceed","line":2695,"in_reply_to":"5f93b717_d8937e5a","updated":"2018-01-30 22:04:31.000000000","message":"Yeah, this value is bogus. The value is supposed to be how many bytes you\u0027d get if you downloaded the submanifest as a SLO. It\u0027s how many bytes a client would see. We\u0027re putting the length of some JSON in here, but that\u0027s got no relation to what a client would see.\n\nIf you modify the test to accurately reflect the bytes on disk, then you get a reasonable failure.\n\nhttps://gist.github.com/smerritt/ced57a97e858fd46ccdd678ba38434d2","commit_id":"d740a9a3d829c276be5cfc2d97d1827e9f50faa5"}],"test/unit/test_locale/eo/LC_MESSAGES/swift.mo":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"607def79afca7f975fe8a9afd9161e6fcd340e63","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"da4df55a_f7cdd431","updated":"2016-12-27 20:28:58.000000000","message":"What changed? Do we know? Was it just that it used to be a symlink?","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"},{"author":{"_account_id":9216,"name":"Joel Wright","email":"joel.wright@gmail.com","username":"jjw-sohonet"},"change_message_id":"16977b64d86c1ab1017f8a4ff15066145abfc1ce","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"7a3c09a3_dd0ce244","in_reply_to":"da4df55a_f7cdd431","updated":"2017-01-13 23:35:22.000000000","message":"Yes, nothing changed here and I spent a while trying to work out what had changed. Good spot with the symlink.","commit_id":"241c39edcf1e1e6fbbea0ae6838718f7d1f09aed"}]}