)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6c0a9992a681e2338963a622d05cd8e02bdf57e8","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Turn stdin uploads into SLO above 10MB."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If the Standard Input upload exceeds 10MB, we should turn it into an"},{"line_number":10,"context_line":"SLO. Below 10MB, a single object is uploaded. This limits the amount"},{"line_number":11,"context_line":"of buffering done when reading from a stream to 10MB. This is actually"},{"line_number":12,"context_line":"tunable as up to 1 segment is buffered, so setting segment-size to a"},{"line_number":13,"context_line":"higher value allows for larger single object uploads from stdin."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"9f436f4f_97589656","line":10,"updated":"2017-08-19 04:44:50.000000000","message":"The justification for the change is not correct. Tim explained to me that the real reason for this patch is that it permits uploads of objects longer than 5GB from stdin. The reduction of buffering is secondary in importance.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":10,"context_line":"SLO. Below 10MB, a single object is uploaded. This limits the amount"},{"line_number":11,"context_line":"of buffering done when reading from a stream to 10MB. This is actually"},{"line_number":12,"context_line":"tunable as up to 1 segment is buffered, so setting segment-size to a"},{"line_number":13,"context_line":"higher value allows for larger single object uploads from stdin."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: Ib13e0b687bc85930c29fe9f151cf96bc53b2e594"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"9f436f4f_3947a6bf","line":13,"updated":"2017-08-18 19:27:11.000000000","message":"Probably worth calling out that this allows uploads of objects \u003e5GiB. That\u0027s the primary impetus here, no?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"}],"swiftclient/service.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1768,"context_line":"                fp.close()"},{"line_number":1769,"context_line":""},{"line_number":1770,"context_line":"    @staticmethod"},{"line_number":1771,"context_line":"    def _put_object(conn, container, name, content, headers, md5\u003dNone):"},{"line_number":1772,"context_line":"        results \u003d {}"},{"line_number":1773,"context_line":"        try:"},{"line_number":1774,"context_line":"            etag \u003d conn.put_object("}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_928b5bbe","line":1771,"updated":"2017-06-28 22:10:40.000000000","message":"I see where you use this twice in _upload_stream_segment - it\u0027s surprising it didn\u0027t already exist - this code is *nearly* duplicated in _create_dir_marker_job, _upload_segment_job and _upload_object_job","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1787,"context_line":"                \u0027error_timestamp\u0027: err_time,"},{"line_number":1788,"context_line":"                \u0027response_dict\u0027: results,"},{"line_number":1789,"context_line":"                \u0027attempts\u0027: conn.attempts"},{"line_number":1790,"context_line":"            }"},{"line_number":1791,"context_line":"        return results"},{"line_number":1792,"context_line":""},{"line_number":1793,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_12b98b77","line":1790,"updated":"2017-06-28 22:10:40.000000000","message":"I think if we could standardize on the shape of these return dicts we\u0027d have a lot more reuse for self._put_object vs. just conn.put_object with ad-hoc filling in of results structure...","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1874,"context_line":""},{"line_number":1875,"context_line":"    @staticmethod"},{"line_number":1876,"context_line":"    def _upload_slo_manifest(conn, segment_results, container, obj,"},{"line_number":1877,"context_line":"                             headers\u003d{}):"},{"line_number":1878,"context_line":"        segment_results.sort(key\u003dlambda di: di[\u0027segment_index\u0027])"},{"line_number":1879,"context_line":"        slo_manifest_paths \u003d set()"},{"line_number":1880,"context_line":"        for seg in segment_results:"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_b2e2d79e","line":1877,"updated":"2017-06-28 22:10:40.000000000","message":"this would have a better chance of reuse with a docstring\n\n:returns: a tuple, the response and... some weird set thing?\n\nHow about just return the response and have one of the keys be the manifest - then you can extract w/e you want..","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1878,"context_line":"        segment_results.sort(key\u003dlambda di: di[\u0027segment_index\u0027])"},{"line_number":1879,"context_line":"        slo_manifest_paths \u003d set()"},{"line_number":1880,"context_line":"        for seg in segment_results:"},{"line_number":1881,"context_line":"            seg_loc \u003d seg[\u0027segment_location\u0027].lstrip(\u0027/\u0027)"},{"line_number":1882,"context_line":"            if isinstance(seg_loc, text_type):"},{"line_number":1883,"context_line":"                seg_loc \u003d seg_loc.encode(\u0027utf-8\u0027)"},{"line_number":1884,"context_line":"            slo_manifest_paths.add(seg_loc)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_12ebcb77","line":1881,"updated":"2017-06-28 22:10:40.000000000","message":"... sorta like how we extract segment_location from the segment_results","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1987,"context_line":"            put_headers.update(split_headers(options[\u0027header\u0027], \u0027\u0027))"},{"line_number":1988,"context_line":""},{"line_number":1989,"context_line":"            # Don\u0027t do segment job if object is not big enough, and never do"},{"line_number":1990,"context_line":"            # a segment job if we\u0027re reading from a stream - we may fail if we"},{"line_number":1991,"context_line":"            # go over the single object limit, but this gives us a nice way"},{"line_number":1992,"context_line":"            # to create objects from memory"},{"line_number":1993,"context_line":"            if (path is not None and segment_size"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_12aceb9c","line":1990,"updated":"2017-06-28 22:10:40.000000000","message":"out-dated comment?","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":2056,"context_line":""},{"line_number":2057,"context_line":"                if options[\u0027use_slo\u0027]:"},{"line_number":2058,"context_line":"                    mr, new_slo_manifest_paths \u003d self._upload_slo_manifest("},{"line_number":2059,"context_line":"                        conn, segment_results, container, obj, put_headers)"},{"line_number":2060,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2061,"context_line":"                else:"},{"line_number":2062,"context_line":"                    new_object_manifest \u003d \u0027%s/%s/%s/%s/%s/\u0027 % ("}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_f2616f06","line":2059,"updated":"2017-06-28 22:10:40.000000000","message":"this is nice... thanks for using the new interface here.","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":2095,"context_line":"                        options,"},{"line_number":2096,"context_line":"                        put_headers,"},{"line_number":2097,"context_line":"                        stream"},{"line_number":2098,"context_line":"                    )"},{"line_number":2099,"context_line":"                    if not ret.get(\u0027success\u0027, True):"},{"line_number":2100,"context_line":"                        err \u003d ClientException("},{"line_number":2101,"context_line":"                            \u0027Failed to upload a segment for %s/%s\u0027 % ("}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_52e263d6","line":2098,"updated":"2017-06-28 22:10:40.000000000","message":"this part is all straight forward and reasonable - KUDOS","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":2143,"context_line":"                            \u0027Object upload verification failed: \u0027"},{"line_number":2144,"context_line":"                            \u0027md5 mismatch, local {0} !\u003d remote {1} \u0027"},{"line_number":2145,"context_line":"                            \u0027(remote object has not been removed)\u0027"},{"line_number":2146,"context_line":"                            .format(contents.get_md5sum(), etag))"},{"line_number":2147,"context_line":"                finally:"},{"line_number":2148,"context_line":"                    if fp is not None:"},{"line_number":2149,"context_line":"                        fp.close()"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_f2784fc4","line":2146,"updated":"2017-06-28 22:10:40.000000000","message":"annoyingly duplicated in _put_object ....","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1785,"context_line":"                fp.close()"},{"line_number":1786,"context_line":""},{"line_number":1787,"context_line":"    @staticmethod"},{"line_number":1788,"context_line":"    def _put_object(conn, container, name, content, headers\u003d{}, md5\u003dNone):"},{"line_number":1789,"context_line":"        \"\"\""},{"line_number":1790,"context_line":"        Upload object into a given container and verify the resulting ETag, if"},{"line_number":1791,"context_line":"        the md5 optional parameter is passed."}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_a827b212","line":1788,"range":{"start_line":1788,"start_character":60,"end_line":1788,"end_character":62},"updated":"2017-07-25 01:10:19.000000000","message":"Let\u0027s not put a mutable data structure in the default args.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1785,"context_line":"                fp.close()"},{"line_number":1786,"context_line":""},{"line_number":1787,"context_line":"    @staticmethod"},{"line_number":1788,"context_line":"    def _put_object(conn, container, name, content, headers\u003d{}, md5\u003dNone):"},{"line_number":1789,"context_line":"        \"\"\""},{"line_number":1790,"context_line":"        Upload object into a given container and verify the resulting ETag, if"},{"line_number":1791,"context_line":"        the md5 optional parameter is passed."}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_e0e7ea11","line":1788,"range":{"start_line":1788,"start_character":60,"end_line":1788,"end_character":62},"in_reply_to":"ff346bd7_a827b212","updated":"2017-07-25 23:35:35.000000000","message":"Done","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1818,"context_line":"                headers\u003dheaders, response_dict\u003dresults)"},{"line_number":1819,"context_line":"            if md5 is not None and etag !\u003d md5:"},{"line_number":1820,"context_line":"                raise SwiftError(\u0027Upload verification failed for {0}: md5 \u0027"},{"line_number":1821,"context_line":"                                 \u0027mismatch {1} !\u003d {2}\u0027.format(name, md5, etag))"},{"line_number":1822,"context_line":"        except Exception as err:"},{"line_number":1823,"context_line":"            traceback, err_time \u003d report_traceback()"},{"line_number":1824,"context_line":"            logger.exception(err)"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_e80dba90","line":1821,"updated":"2017-07-25 01:10:19.000000000","message":"Probably good to keep this, but since we\u0027ve already got the expected etag in hand, may as well do something like\n\n if md5:\n     headers[\u0027etag\u0027] \u003d md5\n\nand let Swift reject the write if it doesn\u0027t match.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1818,"context_line":"                headers\u003dheaders, response_dict\u003dresults)"},{"line_number":1819,"context_line":"            if md5 is not None and etag !\u003d md5:"},{"line_number":1820,"context_line":"                raise SwiftError(\u0027Upload verification failed for {0}: md5 \u0027"},{"line_number":1821,"context_line":"                                 \u0027mismatch {1} !\u003d {2}\u0027.format(name, md5, etag))"},{"line_number":1822,"context_line":"        except Exception as err:"},{"line_number":1823,"context_line":"            traceback, err_time \u003d report_traceback()"},{"line_number":1824,"context_line":"            logger.exception(err)"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_40c0f650","line":1821,"in_reply_to":"ff346bd7_e80dba90","updated":"2017-07-25 23:35:35.000000000","message":"Done","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1861,"context_line":"                    - segment_index - index of the segment"},{"line_number":1862,"context_line":"                    - segment_etag - the ETag for the segment"},{"line_number":1863,"context_line":"        \"\"\""},{"line_number":1864,"context_line":"        buf \u003d br\u0027\u0027"},{"line_number":1865,"context_line":"        dgst \u003d md5()"},{"line_number":1866,"context_line":"        # TODO: Once we are committed to SLO, we no longer should have to"},{"line_number":1867,"context_line":"        # buffer in memory and should be able to use a wrapper around the"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_887aae21","line":1864,"range":{"start_line":1864,"start_character":15,"end_line":1864,"end_character":16},"updated":"2017-07-25 01:10:19.000000000","message":"nit: doesn\u0027t really need to be a raw literal.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1861,"context_line":"                    - segment_index - index of the segment"},{"line_number":1862,"context_line":"                    - segment_etag - the ETag for the segment"},{"line_number":1863,"context_line":"        \"\"\""},{"line_number":1864,"context_line":"        buf \u003d br\u0027\u0027"},{"line_number":1865,"context_line":"        dgst \u003d md5()"},{"line_number":1866,"context_line":"        # TODO: Once we are committed to SLO, we no longer should have to"},{"line_number":1867,"context_line":"        # buffer in memory and should be able to use a wrapper around the"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_00baeedc","line":1864,"range":{"start_line":1864,"start_character":15,"end_line":1864,"end_character":16},"in_reply_to":"ff346bd7_887aae21","updated":"2017-07-25 23:35:35.000000000","message":"Done","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1863,"context_line":"        \"\"\""},{"line_number":1864,"context_line":"        buf \u003d br\u0027\u0027"},{"line_number":1865,"context_line":"        dgst \u003d md5()"},{"line_number":1866,"context_line":"        # TODO: Once we are committed to SLO, we no longer should have to"},{"line_number":1867,"context_line":"        # buffer in memory and should be able to use a wrapper around the"},{"line_number":1868,"context_line":"        # stream."},{"line_number":1869,"context_line":"        while len(buf) \u003c segment_size:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_a8cb3251","line":1866,"range":{"start_line":1866,"start_character":10,"end_line":1866,"end_character":44},"updated":"2017-07-25 01:10:19.000000000","message":"I\u0027m not sure I understand this TODO.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1863,"context_line":"        \"\"\""},{"line_number":1864,"context_line":"        buf \u003d br\u0027\u0027"},{"line_number":1865,"context_line":"        dgst \u003d md5()"},{"line_number":1866,"context_line":"        # TODO: Once we are committed to SLO, we no longer should have to"},{"line_number":1867,"context_line":"        # buffer in memory and should be able to use a wrapper around the"},{"line_number":1868,"context_line":"        # stream."},{"line_number":1869,"context_line":"        while len(buf) \u003c segment_size:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_6e50960b","line":1866,"range":{"start_line":1866,"start_character":10,"end_line":1866,"end_character":44},"in_reply_to":"ff346bd7_a8cb3251","updated":"2017-07-25 23:35:35.000000000","message":"We really need to buffer one segment to see if we\u0027re doing a single file upload or an SLO, once we\u0027re doing SLO, buffering is no longer necessary (unless we want to support retries -- I\u0027m not sure if they would be useful or not) and we could write up to the segment size of bytes out of the standard input stream.\n\nI\u0027ll remove the TODO for now, as I can see it being confusing, but we could consider reworking this if there are concerns over buffering one segment at a time in memory.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1871,"context_line":"            if not data:"},{"line_number":1872,"context_line":"                break"},{"line_number":1873,"context_line":"            dgst.update(data)"},{"line_number":1874,"context_line":"            buf +\u003d data"},{"line_number":1875,"context_line":"        segment_hash \u003d dgst.hexdigest()"},{"line_number":1876,"context_line":""},{"line_number":1877,"context_line":"        if not buf and segment_index \u003e 0:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_88cd0e5f","line":1874,"updated":"2017-07-25 01:10:19.000000000","message":"It\u0027d probably be better to build the buffer as an array, then b\u0027\u0027.join() it at the end, or maybe buffer into a BytesIO. But since fd.read(N) should only return \u003c N bytes when the next read will return none, I\u0027m not too worried.\n\nZaitcev, this sort of thing is why I don\u0027t like messing with the read() contract!","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1871,"context_line":"            if not data:"},{"line_number":1872,"context_line":"                break"},{"line_number":1873,"context_line":"            dgst.update(data)"},{"line_number":1874,"context_line":"            buf +\u003d data"},{"line_number":1875,"context_line":"        segment_hash \u003d dgst.hexdigest()"},{"line_number":1876,"context_line":""},{"line_number":1877,"context_line":"        if not buf and segment_index \u003e 0:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_a0ca026a","line":1874,"in_reply_to":"ff346bd7_88cd0e5f","updated":"2017-07-25 23:35:35.000000000","message":"Done","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1876,"context_line":""},{"line_number":1877,"context_line":"        if not buf and segment_index \u003e 0:"},{"line_number":1878,"context_line":"            # Happens if the segment size aligns with the object size"},{"line_number":1879,"context_line":"            return {\u0027complete\u0027: True, \u0027segment_size\u0027: 0}"},{"line_number":1880,"context_line":""},{"line_number":1881,"context_line":"        if segment_index \u003d\u003d 0 and len(buf) \u003c segment_size:"},{"line_number":1882,"context_line":"            ret \u003d SwiftService._put_object("}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_68f30aa0","line":1879,"updated":"2017-07-25 01:10:19.000000000","message":"Given the comment on _upload_slo_manifest, we\u0027re missing\n\n* segment_location\n* segment_etag\n* segment_index","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1876,"context_line":""},{"line_number":1877,"context_line":"        if not buf and segment_index \u003e 0:"},{"line_number":1878,"context_line":"            # Happens if the segment size aligns with the object size"},{"line_number":1879,"context_line":"            return {\u0027complete\u0027: True, \u0027segment_size\u0027: 0}"},{"line_number":1880,"context_line":""},{"line_number":1881,"context_line":"        if segment_index \u003d\u003d 0 and len(buf) \u003c segment_size:"},{"line_number":1882,"context_line":"            ret \u003d SwiftService._put_object("}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_2ef60efb","line":1879,"in_reply_to":"ff346bd7_68f30aa0","updated":"2017-07-25 23:35:35.000000000","message":"Those don\u0027t really make sense here, as all would be None. I\u0027m not convinced using these dictionary structures is best regardless, but the code handling this response doesn\u0027t append the value for 0-length segments.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1965,"context_line":"        for seg in segment_results:"},{"line_number":1966,"context_line":"            seg_loc \u003d seg[\u0027segment_location\u0027].lstrip(\u0027/\u0027)"},{"line_number":1967,"context_line":"            if isinstance(seg_loc, text_type):"},{"line_number":1968,"context_line":"                seg_loc \u003d seg_loc.encode(\u0027utf-8\u0027)"},{"line_number":1969,"context_line":"            segment_paths.add(seg_loc)"},{"line_number":1970,"context_line":""},{"line_number":1971,"context_line":"        manifest_data \u003d json.dumps(["}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_eada4f15","line":1968,"updated":"2017-07-25 01:10:19.000000000","message":"What\u0027s the usual type for strings in result dicts? I was kind of expecting native strings, but I wouldn\u0027t be at all surprised if it was \"whatever we feel like at the time\"...","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1965,"context_line":"        for seg in segment_results:"},{"line_number":1966,"context_line":"            seg_loc \u003d seg[\u0027segment_location\u0027].lstrip(\u0027/\u0027)"},{"line_number":1967,"context_line":"            if isinstance(seg_loc, text_type):"},{"line_number":1968,"context_line":"                seg_loc \u003d seg_loc.encode(\u0027utf-8\u0027)"},{"line_number":1969,"context_line":"            segment_paths.add(seg_loc)"},{"line_number":1970,"context_line":""},{"line_number":1971,"context_line":"        manifest_data \u003d json.dumps(["}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_8ed29a5b","line":1968,"in_reply_to":"ff346bd7_eada4f15","updated":"2017-07-25 23:35:35.000000000","message":"I don\u0027t know what the contract is :\\","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":2165,"context_line":"                segment \u003d 0"},{"line_number":2166,"context_line":"                results \u003d []"},{"line_number":2167,"context_line":"                manifest \u003d True"},{"line_number":2168,"context_line":"                while True:"},{"line_number":2169,"context_line":"                    segment_name \u003d \u0027%s/slo/%s/%s/%08d\u0027 % ("},{"line_number":2170,"context_line":"                        obj, put_headers[\u0027x-object-meta-mtime\u0027],"},{"line_number":2171,"context_line":"                        segment_size, segment"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_6a83ffcd","line":2168,"updated":"2017-07-25 01:10:19.000000000","message":"nit: Maybe\n\n for segment in itertools.count():\n\n? *shrug*","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":2190,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2191,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2192,"context_line":"                        })"},{"line_number":2193,"context_line":"                        return res"},{"line_number":2194,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_8acaa317","line":2193,"updated":"2017-07-25 01:10:19.000000000","message":"Should we do any sort of cleanup before returning?\n\nDoes the caller (or ultimately, CLI user) know what segments are out there?","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"190013da1ab4acee78198432c727369b136e3498","unresolved":false,"context_lines":[{"line_number":2190,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2191,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2192,"context_line":"                        })"},{"line_number":2193,"context_line":"                        return res"},{"line_number":2194,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_bb6fd790","line":2193,"in_reply_to":"ff346bd7_8acaa317","updated":"2017-07-25 21:59:09.000000000","message":"Hmmm....\n\n $ dd if\u003d/dev/zero | swift upload c - --object-name foo\n ^C Aborted\n 163969+0 records in\n 163968+0 records out\n 83951616 bytes transferred in 2.484894 secs (33784790 bytes/sec)\n\n $ swift list\n c\n c_segments\n\nGood...\n\n $ swift list c_segments\n foo/slo/1501017559.000000/10485760/00000000\n foo/slo/1501017559.000000/10485760/00000001\n foo/slo/1501017559.000000/10485760/00000002\n foo/slo/1501017559.000000/10485760/00000003\n foo/slo/1501017559.000000/10485760/00000004\n foo/slo/1501017559.000000/10485760/00000005\n foo/slo/1501017559.000000/10485760/00000006\n foo/slo/1501017559.000000/10485760/00000007\n\nCool...\n\n $ swift list c\n\n:-( Nothing?","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":2190,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2191,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2192,"context_line":"                        })"},{"line_number":2193,"context_line":"                        return res"},{"line_number":2194,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_8e373ab1","line":2193,"in_reply_to":"ff346bd7_8acaa317","updated":"2017-07-25 23:35:35.000000000","message":"This is the same as the error handling on line 1994.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":2190,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2191,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2192,"context_line":"                        })"},{"line_number":2193,"context_line":"                        return res"},{"line_number":2194,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_b013715c","line":2193,"in_reply_to":"ff346bd7_bb6fd790","updated":"2017-07-25 23:35:35.000000000","message":"I think this makes sense as we never hit EOF?","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"},{"line_number":2197,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2198,"context_line":"                        if segment \u003d\u003d 1:"},{"line_number":2199,"context_line":"                            manifest \u003d False"},{"line_number":2200,"context_line":"                        break"},{"line_number":2201,"context_line":"                if manifest:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_4aa27b4d","line":2198,"range":{"start_line":2198,"start_character":24,"end_line":2198,"end_character":40},"updated":"2017-07-25 01:10:19.000000000","message":"We\u0027ve embedded this check twice, here and line 1881 -- maybe better to use a separate key to indicate that we\u0027ve already satisfied the PUT?","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":2195,"context_line":"                        results.append(ret)"},{"line_number":2196,"context_line":"                    segment +\u003d 1"},{"line_number":2197,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2198,"context_line":"                        if segment \u003d\u003d 1:"},{"line_number":2199,"context_line":"                            manifest \u003d False"},{"line_number":2200,"context_line":"                        break"},{"line_number":2201,"context_line":"                if manifest:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_e35c917f","line":2198,"range":{"start_line":2198,"start_character":24,"end_line":2198,"end_character":40},"in_reply_to":"ff346bd7_4aa27b4d","updated":"2017-07-25 23:35:35.000000000","message":"We can actually check the segment_location -- I\u0027ll do that.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":1503,"context_line":"                # We\u0027ve got a file like object to upload to o"},{"line_number":1504,"context_line":"                file_future \u003d self.thread_manager.object_uu_pool.submit("},{"line_number":1505,"context_line":"                    self._upload_object_job, container, s, o, object_options,"},{"line_number":1506,"context_line":"                    results_queue\u003drq"},{"line_number":1507,"context_line":"                )"},{"line_number":1508,"context_line":"                details[\u0027file\u0027] \u003d s"},{"line_number":1509,"context_line":"                details[\u0027object\u0027] \u003d o"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_4ad46548","line":1506,"updated":"2017-07-27 00:38:51.000000000","message":"? Oh... to plumb the results back out to shell so it can render the segments, I suppose...\n\nOnly that still doesn\u0027t happen:\n\n $ dd if\u003d/dev/zero bs\u003d10240000 count\u003d5 | swift upload c - --object-name foo\n 5+0 records in\n 5+0 records out\n 51200000 bytes transferred in 1.455955 secs (35165921 bytes/sec)\n foo","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":1503,"context_line":"                # We\u0027ve got a file like object to upload to o"},{"line_number":1504,"context_line":"                file_future \u003d self.thread_manager.object_uu_pool.submit("},{"line_number":1505,"context_line":"                    self._upload_object_job, container, s, o, object_options,"},{"line_number":1506,"context_line":"                    results_queue\u003drq"},{"line_number":1507,"context_line":"                )"},{"line_number":1508,"context_line":"                details[\u0027file\u0027] \u003d s"},{"line_number":1509,"context_line":"                details[\u0027object\u0027] \u003d o"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_5dbaca6d","line":1506,"in_reply_to":"ff346bd7_4ad46548","updated":"2017-07-27 22:38:45.000000000","message":"I was missing the \"for_object\" field in the return dictionary.","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":1899,"context_line":"            ret[\u0027segment_location\u0027] \u003d \u0027/%s/%s\u0027 % ("},{"line_number":1900,"context_line":"                segment_container, segment_name)"},{"line_number":1901,"context_line":""},{"line_number":1902,"context_line":"        ret[\u0027success\u0027] \u003d True"},{"line_number":1903,"context_line":"        ret[\u0027complete\u0027] \u003d len(buf) \u003c segment_size"},{"line_number":1904,"context_line":"        ret[\u0027segment_size\u0027] \u003d len(buf)"},{"line_number":1905,"context_line":"        ret[\u0027segment_index\u0027] \u003d segment_index"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_cadcd5e5","line":1902,"range":{"start_line":1902,"start_character":8,"end_line":1902,"end_character":29},"updated":"2017-07-27 00:38:51.000000000","message":"That doesn\u0027t seem right. Trying to upload when I don\u0027t have access to the segments container, I see\n\n $ dd if\u003d/dev/zero bs\u003d10240000 count\u003d5 | swift upload c - --object-name foo\n Warning: failed to create container \u0027c\u0027: 403 Forbidden: \u003chtml\u003e\u003ch1\u003eForbidden\u003c/h1\u003e\u003cp\u003eAccess was denied to this resourc\n Warning: failed to create container \u0027c_segments\u0027: 403 Forbidden: \u003chtml\u003e\u003ch1\u003eForbidden\u003c/h1\u003e\u003cp\u003eAccess was denied to this resourc\n 5+0 records in\n 5+0 records out\n 51200000 bytes transferred in 0.518844 secs (98680935 bytes/sec)\n Object PUT failed: http://saio:8080/v1/AUTH_test/c/foo?multipart-manifest\u003dput 400 Bad Request  [first 60 chars of response] Errors:\n /c_segments/foo/slo/1501115011.000000/10485760/00000\n\nwith no mention of the segment failure at all.","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":1899,"context_line":"            ret[\u0027segment_location\u0027] \u003d \u0027/%s/%s\u0027 % ("},{"line_number":1900,"context_line":"                segment_container, segment_name)"},{"line_number":1901,"context_line":""},{"line_number":1902,"context_line":"        ret[\u0027success\u0027] \u003d True"},{"line_number":1903,"context_line":"        ret[\u0027complete\u0027] \u003d len(buf) \u003c segment_size"},{"line_number":1904,"context_line":"        ret[\u0027segment_size\u0027] \u003d len(buf)"},{"line_number":1905,"context_line":"        ret[\u0027segment_index\u0027] \u003d segment_index"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_f8ff4ca8","line":1902,"range":{"start_line":1902,"start_character":8,"end_line":1902,"end_character":29},"in_reply_to":"ff346bd7_cadcd5e5","updated":"2017-07-27 22:38:45.000000000","message":"Good catch -- done.","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":2197,"context_line":"                        stream"},{"line_number":2198,"context_line":"                    )"},{"line_number":2199,"context_line":"                    if not ret[\u0027success\u0027]:"},{"line_number":2200,"context_line":"                        err \u003d ClientException("},{"line_number":2201,"context_line":"                            \u0027Failed to upload a segment for %s/%s\u0027 % ("},{"line_number":2202,"context_line":"                                container, obj))"},{"line_number":2203,"context_line":"                        ret.update({"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_8a7ecdac","line":2200,"range":{"start_line":2200,"start_character":24,"end_line":2200,"end_character":27},"updated":"2017-07-27 00:38:51.000000000","message":"Do we ever do anything with this?","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":2197,"context_line":"                        stream"},{"line_number":2198,"context_line":"                    )"},{"line_number":2199,"context_line":"                    if not ret[\u0027success\u0027]:"},{"line_number":2200,"context_line":"                        err \u003d ClientException("},{"line_number":2201,"context_line":"                            \u0027Failed to upload a segment for %s/%s\u0027 % ("},{"line_number":2202,"context_line":"                                container, obj))"},{"line_number":2203,"context_line":"                        ret.update({"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_18e57009","line":2200,"range":{"start_line":2200,"start_character":24,"end_line":2200,"end_character":27},"in_reply_to":"ff346bd7_8a7ecdac","updated":"2017-07-27 22:38:45.000000000","message":"Done","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":2200,"context_line":"                        err \u003d ClientException("},{"line_number":2201,"context_line":"                            \u0027Failed to upload a segment for %s/%s\u0027 % ("},{"line_number":2202,"context_line":"                                container, obj))"},{"line_number":2203,"context_line":"                        ret.update({"},{"line_number":2204,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2205,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2206,"context_line":"                        })"},{"line_number":2207,"context_line":"                        return ret"},{"line_number":2208,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2209,"context_line":"                        results.append(ret)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_4a07c5cb","line":2206,"range":{"start_line":2203,"start_character":24,"end_line":2206,"end_character":26},"updated":"2017-07-27 00:38:51.000000000","message":"Seems unnecessary...","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":2200,"context_line":"                        err \u003d ClientException("},{"line_number":2201,"context_line":"                            \u0027Failed to upload a segment for %s/%s\u0027 % ("},{"line_number":2202,"context_line":"                                container, obj))"},{"line_number":2203,"context_line":"                        ret.update({"},{"line_number":2204,"context_line":"                            \u0027success\u0027: False,"},{"line_number":2205,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2206,"context_line":"                        })"},{"line_number":2207,"context_line":"                        return ret"},{"line_number":2208,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2209,"context_line":"                        results.append(ret)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_38e834e0","line":2206,"range":{"start_line":2203,"start_character":24,"end_line":2206,"end_character":26},"in_reply_to":"ff346bd7_4a07c5cb","updated":"2017-07-27 22:38:45.000000000","message":"Done","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":2205,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2206,"context_line":"                        })"},{"line_number":2207,"context_line":"                        return ret"},{"line_number":2208,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2209,"context_line":"                        results.append(ret)"},{"line_number":2210,"context_line":"                    segment +\u003d 1"},{"line_number":2211,"context_line":"                    if results_queue is not None:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_4a1c6577","line":2208,"updated":"2017-07-27 00:38:51.000000000","message":"I think we want\n\n if segment \u003d\u003d 0 or ...:","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":2205,"context_line":"                            \u0027error\u0027: ret[\u0027error\u0027]"},{"line_number":2206,"context_line":"                        })"},{"line_number":2207,"context_line":"                        return ret"},{"line_number":2208,"context_line":"                    if ret[\u0027segment_size\u0027] \u003e 0:"},{"line_number":2209,"context_line":"                        results.append(ret)"},{"line_number":2210,"context_line":"                    segment +\u003d 1"},{"line_number":2211,"context_line":"                    if results_queue is not None:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_38fb9479","line":2208,"in_reply_to":"ff346bd7_4a1c6577","updated":"2017-07-27 22:38:45.000000000","message":"This is where we fail to store the 0-sized object PUT results. We should check for segment \u003d\u003d 0 and complete OR segment_size \u003e 0. Done.","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8e0a5772de5ebd6dc8cda292b4f2c4e234cf1dc3","unresolved":false,"context_lines":[{"line_number":2212,"context_line":"                        results_queue.put(ret)"},{"line_number":2213,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2214,"context_line":"                        break"},{"line_number":2215,"context_line":"                if results[0][\u0027segment_location\u0027] !\u003d \u0027/%s/%s\u0027 % ("},{"line_number":2216,"context_line":"                        container, obj):"},{"line_number":2217,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2218,"context_line":"                        conn, results, container, obj, put_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_2a0da1a0","line":2215,"updated":"2017-07-27 00:38:51.000000000","message":"Hit an IndexError here with a zero-byte file. Object still got uploaded though...","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"7a973aaea493a7009a96f3e5372e25fcfe167b14","unresolved":false,"context_lines":[{"line_number":2212,"context_line":"                        results_queue.put(ret)"},{"line_number":2213,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2214,"context_line":"                        break"},{"line_number":2215,"context_line":"                if results[0][\u0027segment_location\u0027] !\u003d \u0027/%s/%s\u0027 % ("},{"line_number":2216,"context_line":"                        container, obj):"},{"line_number":2217,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2218,"context_line":"                        conn, results, container, obj, put_headers)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff346bd7_f8d74ced","line":2215,"in_reply_to":"ff346bd7_2a0da1a0","updated":"2017-07-27 22:38:45.000000000","message":"Done","commit_id":"92a14a5f96c5f8c510988a371e6a7edd435787a8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6c0a9992a681e2338963a622d05cd8e02bdf57e8","unresolved":false,"context_lines":[{"line_number":1828,"context_line":"                                 \u0027mismatch {1} !\u003d {2}\u0027.format(name, md5, etag))"},{"line_number":1829,"context_line":"            results[\u0027success\u0027] \u003d True"},{"line_number":1830,"context_line":"        except Exception as err:"},{"line_number":1831,"context_line":"            traceback, err_time \u003d report_traceback()"},{"line_number":1832,"context_line":"            logger.exception(err)"},{"line_number":1833,"context_line":"            return {"},{"line_number":1834,"context_line":"                \u0027success\u0027: False,"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_f73ec2d8","line":1831,"updated":"2017-08-19 04:44:50.000000000","message":"I may be blind, but isn\u0027t the \"traceback\" variable unused?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":1828,"context_line":"                                 \u0027mismatch {1} !\u003d {2}\u0027.format(name, md5, etag))"},{"line_number":1829,"context_line":"            results[\u0027success\u0027] \u003d True"},{"line_number":1830,"context_line":"        except Exception as err:"},{"line_number":1831,"context_line":"            traceback, err_time \u003d report_traceback()"},{"line_number":1832,"context_line":"            logger.exception(err)"},{"line_number":1833,"context_line":"            return {"},{"line_number":1834,"context_line":"                \u0027success\u0027: False,"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_7de27503","line":1831,"in_reply_to":"9f436f4f_f73ec2d8","updated":"2017-09-14 22:43:24.000000000","message":"Done","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":1836,"context_line":"                \u0027error_timestamp\u0027: err_time,"},{"line_number":1837,"context_line":"                \u0027response_dict\u0027: results,"},{"line_number":1838,"context_line":"                \u0027attempts\u0027: conn.attempts"},{"line_number":1839,"context_line":"            }"},{"line_number":1840,"context_line":"        return results"},{"line_number":1841,"context_line":""},{"line_number":1842,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_9d1afd14","line":1839,"updated":"2017-08-18 19:27:11.000000000","message":"These usually include a \u0027traceback\u0027 key, too...","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":1836,"context_line":"                \u0027error_timestamp\u0027: err_time,"},{"line_number":1837,"context_line":"                \u0027response_dict\u0027: results,"},{"line_number":1838,"context_line":"                \u0027attempts\u0027: conn.attempts"},{"line_number":1839,"context_line":"            }"},{"line_number":1840,"context_line":"        return results"},{"line_number":1841,"context_line":""},{"line_number":1842,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_5df1f9cc","line":1839,"in_reply_to":"9f436f4f_9d1afd14","updated":"2017-09-14 22:43:24.000000000","message":"Done","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6c0a9992a681e2338963a622d05cd8e02bdf57e8","unresolved":false,"context_lines":[{"line_number":1847,"context_line":"        \"\"\""},{"line_number":1848,"context_line":"        Upload a segment from a stream, buffering it in memory first. The"},{"line_number":1849,"context_line":"        resulting object is placed either as a segment in the segment"},{"line_number":1850,"context_line":"        container, or if it is smaller than a single segment, as the give"},{"line_number":1851,"context_line":"        object name."},{"line_number":1852,"context_line":""},{"line_number":1853,"context_line":"        :param conn: Swift Connection to use."}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_97065620","line":1850,"updated":"2017-08-19 04:44:50.000000000","message":"Probebly \"giveN object name\" (unimportant nit, but...)","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":1847,"context_line":"        \"\"\""},{"line_number":1848,"context_line":"        Upload a segment from a stream, buffering it in memory first. The"},{"line_number":1849,"context_line":"        resulting object is placed either as a segment in the segment"},{"line_number":1850,"context_line":"        container, or if it is smaller than a single segment, as the give"},{"line_number":1851,"context_line":"        object name."},{"line_number":1852,"context_line":""},{"line_number":1853,"context_line":"        :param conn: Swift Connection to use."}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_3decedf1","line":1850,"in_reply_to":"9f436f4f_97065620","updated":"2017-09-14 22:43:24.000000000","message":"Done","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6c0a9992a681e2338963a622d05cd8e02bdf57e8","unresolved":false,"context_lines":[{"line_number":1877,"context_line":"                break"},{"line_number":1878,"context_line":"            dgst.update(data)"},{"line_number":1879,"context_line":"            buf.append(data)"},{"line_number":1880,"context_line":"        buf \u003d b\u0027\u0027.join(buf)"},{"line_number":1881,"context_line":"        segment_hash \u003d dgst.hexdigest()"},{"line_number":1882,"context_line":""},{"line_number":1883,"context_line":"        if not buf and segment_index \u003e 0:"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_32c218eb","line":1880,"updated":"2017-08-19 04:44:50.000000000","message":"Is it necessary to join these? A list is iterable and put_object() appears to accept those. Wanna save some more buffering?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":1877,"context_line":"                break"},{"line_number":1878,"context_line":"            dgst.update(data)"},{"line_number":1879,"context_line":"            buf.append(data)"},{"line_number":1880,"context_line":"        buf \u003d b\u0027\u0027.join(buf)"},{"line_number":1881,"context_line":"        segment_hash \u003d dgst.hexdigest()"},{"line_number":1882,"context_line":""},{"line_number":1883,"context_line":"        if not buf and segment_index \u003e 0:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_1dbdb1d7","line":1880,"in_reply_to":"9f436f4f_32c218eb","updated":"2017-09-14 22:43:24.000000000","message":"TL;DR: we have to do this. Below is a more thorough explanation of why -- it\u0027s not because of swiftclient, but because of the requests library that it uses to ultimately submit the request.\n\nswiftclient submits the request using requests.Session.request().\n\nPassing a list of strings will fail with a \"too many values to unpack error\" (that part is really not great), as it calls the following function to encode the content:\n\ndef to_key_val_list(value):\n    \"\"\"Take an object and test to see if it can be represented as a\n    dictionary. If it can be, return a list of tuples, e.g.,\n\n    ::\n\n        \u003e\u003e\u003e to_key_val_list([(\u0027key\u0027, \u0027val\u0027)])\n        [(\u0027key\u0027, \u0027val\u0027)]\n        \u003e\u003e\u003e to_key_val_list({\u0027key\u0027: \u0027val\u0027})\n        [(\u0027key\u0027, \u0027val\u0027)]\n        \u003e\u003e\u003e to_key_val_list(\u0027string\u0027)\n        ValueError: cannot encode objects that are not 2-tuples.\n\n    :rtype: list\n    \"\"\"\n    if value is None:\n        return None\n\n    if isinstance(value, (str, bytes, bool, int)):\n        raise ValueError(\u0027cannot encode objects that are not 2-tuples\u0027)\n\n    if isinstance(value, collections.Mapping):\n        value \u003d value.items()\n\n    return list(value)\n\nThe call to it is through the _encode_params() method (defined in the RequestEncodingMixin), called by the prepare_body() method of the PreparedRequest  call in the prepare() method of the Request class (the number of layers of indirection here is pretty amazing!).\n\nUnless we mutate requests to properly handle a list of bytes, we have to do the concatenation. I agree that it would be great to avoid it.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":1975,"context_line":"        :param obj: The name of the manifest object to use."},{"line_number":1976,"context_line":"        :param headers: Optional set of headers to attach to the manifest."},{"line_number":1977,"context_line":"        \"\"\""},{"line_number":1978,"context_line":"        if headers is None:"},{"line_number":1979,"context_line":"            headers \u003d {}"},{"line_number":1980,"context_line":"        segment_results.sort(key\u003dlambda di: di[\u0027segment_index\u0027])"},{"line_number":1981,"context_line":"        segment_paths \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_394e86c1","line":1978,"updated":"2017-08-18 19:27:11.000000000","message":"Will this ever be true? Should we just make it required? Tests still pass when I apply\n\n diff --git a/swiftclient/service.py b/swiftclient/service.py\n index 07d101c..7b83144 100644\n --- a/swiftclient/service.py\n +++ b/swiftclient/service.py\n @@ -1958,8 +1958,7 @@ class SwiftService(object):\n              return not fp.read(1)\n\n      @staticmethod\n -    def _upload_slo_manifest(conn, segment_results, container, obj,\n -                             headers\u003dNone):\n +    def _upload_slo_manifest(conn, segment_results, container, obj, headers):\n          \"\"\"\n          Upload an SLO manifest, given the results of uploading each segment, to\n          the specified container.\n @@ -1975,8 +1974,6 @@ class SwiftService(object):\n          :param obj: The name of the manifest object to use.\n          :param headers: Optional set of headers to attach to the manifest.\n          \"\"\"\n -        if headers is None:\n -            headers \u003d {}\n          segment_results.sort(key\u003dlambda di: di[\u0027segment_index\u0027])\n          segment_paths \u003d set()\n          for seg in segment_results:","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":1975,"context_line":"        :param obj: The name of the manifest object to use."},{"line_number":1976,"context_line":"        :param headers: Optional set of headers to attach to the manifest."},{"line_number":1977,"context_line":"        \"\"\""},{"line_number":1978,"context_line":"        if headers is None:"},{"line_number":1979,"context_line":"            headers \u003d {}"},{"line_number":1980,"context_line":"        segment_results.sort(key\u003dlambda di: di[\u0027segment_index\u0027])"},{"line_number":1981,"context_line":"        segment_paths \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_1da251af","line":1978,"in_reply_to":"9f436f4f_394e86c1","updated":"2017-09-14 22:43:24.000000000","message":"Done","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":1999,"context_line":"            headers\u003dheaders,"},{"line_number":2000,"context_line":"            query_string\u003d\u0027multipart-manifest\u003dput\u0027,"},{"line_number":2001,"context_line":"            response_dict\u003dresponse)"},{"line_number":2002,"context_line":"        response[\u0027segment_paths\u0027] \u003d segment_paths"},{"line_number":2003,"context_line":"        return response"},{"line_number":2004,"context_line":""},{"line_number":2005,"context_line":"    def _upload_object_job(self, conn, container, source, obj, options,"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_79d9aef2","line":2002,"updated":"2017-08-18 19:27:11.000000000","message":"OK, new key in the response dict...","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":2178,"context_line":"                        response_dict\u003dmr"},{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_79876eb8","line":2181,"range":{"start_line":2181,"start_character":46,"end_line":2181,"end_character":50},"updated":"2017-08-18 19:27:11.000000000","message":"Can path ever be empty? Should we check for `path is None` instead?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2178,"context_line":"                        response_dict\u003dmr"},{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_bd70dddd","line":2181,"range":{"start_line":2181,"start_character":46,"end_line":2181,"end_character":50},"in_reply_to":"9f436f4f_79876eb8","updated":"2017-09-14 22:43:24.000000000","message":"Done","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6c0a9992a681e2338963a622d05cd8e02bdf57e8","unresolved":false,"context_lines":[{"line_number":2178,"context_line":"                        response_dict\u003dmr"},{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_a63c8dbf","line":2181,"range":{"start_line":2181,"start_character":46,"end_line":2181,"end_character":50},"in_reply_to":"9f436f4f_79876eb8","updated":"2017-08-19 04:44:50.000000000","message":"I expected something like this:\n\nstat.S_ISFIFO(os.fstat(fd.fileno()).st_mode)","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2178,"context_line":"                        response_dict\u003dmr"},{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_3d2aad8f","line":2181,"range":{"start_line":2181,"start_character":46,"end_line":2181,"end_character":50},"in_reply_to":"9f436f4f_a63c8dbf","updated":"2017-09-14 22:43:24.000000000","message":"The control flow is a little convoluted with the way `path` is defined, where it\u0027s based on the check of `source`. Something akin to the above suggestion could be done on line 2015, possibly.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"},{"line_number":2185,"context_line":"                    segment_name \u003d \u0027%s/slo/%s/%s/%08d\u0027 % ("}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_9d1b1d72","line":2182,"updated":"2017-08-18 19:27:11.000000000","message":"We could replace segment with len(results) ... but maybe this is more clear?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2179,"context_line":"                    )"},{"line_number":2180,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2181,"context_line":"            elif \u0027use_slo\u0027 in options and not path:"},{"line_number":2182,"context_line":"                segment \u003d 0"},{"line_number":2183,"context_line":"                results \u003d []"},{"line_number":2184,"context_line":"                while True:"},{"line_number":2185,"context_line":"                    segment_name \u003d \u0027%s/slo/%s/%s/%08d\u0027 % ("}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_fd1d4573","line":2182,"in_reply_to":"9f436f4f_9d1b1d72","updated":"2017-09-14 22:43:24.000000000","message":"I think that\u0027d be more confusing.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":2184,"context_line":"                while True:"},{"line_number":2185,"context_line":"                    segment_name \u003d \u0027%s/slo/%s/%s/%08d\u0027 % ("},{"line_number":2186,"context_line":"                        obj, put_headers[\u0027x-object-meta-mtime\u0027],"},{"line_number":2187,"context_line":"                        segment_size, segment"},{"line_number":2188,"context_line":"                    )"},{"line_number":2189,"context_line":"                    seg_container \u003d container + \u0027_segments\u0027"},{"line_number":2190,"context_line":"                    if options[\u0027segment_container\u0027]:"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_d94f7a90","line":2187,"updated":"2017-08-18 19:27:11.000000000","message":"Interesting... so we\u0027re changing our segment naming conventions for these? Ah, because we don\u0027t know the full size...","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2184,"context_line":"                while True:"},{"line_number":2185,"context_line":"                    segment_name \u003d \u0027%s/slo/%s/%s/%08d\u0027 % ("},{"line_number":2186,"context_line":"                        obj, put_headers[\u0027x-object-meta-mtime\u0027],"},{"line_number":2187,"context_line":"                        segment_size, segment"},{"line_number":2188,"context_line":"                    )"},{"line_number":2189,"context_line":"                    seg_container \u003d container + \u0027_segments\u0027"},{"line_number":2190,"context_line":"                    if options[\u0027segment_container\u0027]:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_9d0e4139","line":2187,"in_reply_to":"9f436f4f_d94f7a90","updated":"2017-09-14 22:43:24.000000000","message":"Yep, I looked into unifying these two paths, but didn\u0027t see a great way forward.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":2205,"context_line":"                        results.append(ret)"},{"line_number":2206,"context_line":"                    segment +\u003d 1"},{"line_number":2207,"context_line":"                    if results_queue is not None:"},{"line_number":2208,"context_line":"                        results_queue.put(ret)"},{"line_number":2209,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2210,"context_line":"                        break"},{"line_number":2211,"context_line":"                if results[0][\u0027segment_location\u0027] !\u003d \u0027/%s/%s\u0027 % ("}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_5d4c9575","line":2208,"updated":"2017-08-18 19:27:11.000000000","message":"Do we include trailing zero-byte completes here? Experimentally I don\u0027t see one in the CLI output... where does it get filtered?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2205,"context_line":"                        results.append(ret)"},{"line_number":2206,"context_line":"                    segment +\u003d 1"},{"line_number":2207,"context_line":"                    if results_queue is not None:"},{"line_number":2208,"context_line":"                        results_queue.put(ret)"},{"line_number":2209,"context_line":"                    if ret[\u0027complete\u0027]:"},{"line_number":2210,"context_line":"                        break"},{"line_number":2211,"context_line":"                if results[0][\u0027segment_location\u0027] !\u003d \u0027/%s/%s\u0027 % ("}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_d4e804e3","line":2208,"in_reply_to":"9f436f4f_5d4c9575","updated":"2017-09-14 22:43:24.000000000","message":"Reworked the logic here a bit to fix up the notifications.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3efc1f6cf896ed207fc877d665f7ff52e79f2a5e","unresolved":false,"context_lines":[{"line_number":2213,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2214,"context_line":"                        conn, results, container, obj, put_headers)"},{"line_number":2215,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d response"},{"line_number":2216,"context_line":"                    new_slo_manifest_paths \u003d response[\u0027segment_paths\u0027]"},{"line_number":2217,"context_line":"                else:"},{"line_number":2218,"context_line":"                    res[\u0027response_dict\u0027] \u003d ret"},{"line_number":2219,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":9,"id":"9f436f4f_fda8e9db","line":2216,"updated":"2017-08-18 19:27:11.000000000","message":"Why not just use\n\n {res[\u0027segment_location\u0027] for res in results}\n\n? Then we could avoid the new key in the response dict, no?\n\nDo we have any stdin-specific tests where we freeze time, do an overwrite, and verify that we don\u0027t try to remove the segments we just uploaded?","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"48ad9bdc59fcf4297c60fafe63d119a0fa745678","unresolved":false,"context_lines":[{"line_number":2213,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2214,"context_line":"                        conn, results, container, obj, put_headers)"},{"line_number":2215,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d response"},{"line_number":2216,"context_line":"                    new_slo_manifest_paths \u003d response[\u0027segment_paths\u0027]"},{"line_number":2217,"context_line":"                else:"},{"line_number":2218,"context_line":"                    res[\u0027response_dict\u0027] \u003d ret"},{"line_number":2219,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f2577a7_c05dfee9","line":2216,"in_reply_to":"9f436f4f_fda8e9db","updated":"2017-09-14 22:43:24.000000000","message":"Yep, makes sense. I think we can do the same for the regular SLO upload... but I won\u0027t clean that up in this patch.","commit_id":"c515d91cf2d006d17e58b481ef770c3c97c8f98d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"35d93ba4c76cd410ec590a06099d4f487d053308","unresolved":false,"context_lines":[{"line_number":1873,"context_line":"        buf \u003d []"},{"line_number":1874,"context_line":"        dgst \u003d md5()"},{"line_number":1875,"context_line":"        while sum(map(len, buf)) \u003c segment_size:"},{"line_number":1876,"context_line":"            data \u003d fd.read(segment_size - len(buf))"},{"line_number":1877,"context_line":"            if not data:"},{"line_number":1878,"context_line":"                break"},{"line_number":1879,"context_line":"            dgst.update(data)"}],"source_content_type":"text/x-python","patch_set":12,"id":"7f96bb07_0792319f","line":1876,"range":{"start_line":1876,"start_character":42,"end_line":1876,"end_character":50},"updated":"2018-01-13 02:34:43.000000000","message":"I think we want this to be\n\n sum(map(len, buf))","commit_id":"7a5422716d1b7581cb949014d4a33ea28476f1bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"90e6557fe671cf3a4ddfa979a3a186bfb1e63a71","unresolved":false,"context_lines":[{"line_number":1999,"context_line":"            headers\u003dheaders,"},{"line_number":2000,"context_line":"            query_string\u003d\u0027multipart-manifest\u003dput\u0027,"},{"line_number":2001,"context_line":"            response_dict\u003dresponse)"},{"line_number":2002,"context_line":"        response[\u0027segment_paths\u0027] \u003d segment_paths"},{"line_number":2003,"context_line":"        return response"},{"line_number":2004,"context_line":""},{"line_number":2005,"context_line":"    def _upload_object_job(self, conn, container, source, obj, options,"}],"source_content_type":"text/x-python","patch_set":12,"id":"7f96bb07_11ad9387","line":2002,"updated":"2018-01-13 00:29:30.000000000","message":"Is this still necessary?","commit_id":"7a5422716d1b7581cb949014d4a33ea28476f1bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"90e6557fe671cf3a4ddfa979a3a186bfb1e63a71","unresolved":false,"context_lines":[{"line_number":2160,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2161,"context_line":"                        conn, segment_results, container, obj, put_headers)"},{"line_number":2162,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d response"},{"line_number":2163,"context_line":"                    new_slo_manifest_paths \u003d response[\u0027segment_paths\u0027]"},{"line_number":2164,"context_line":"                else:"},{"line_number":2165,"context_line":"                    new_object_manifest \u003d \u0027%s/%s/%s/%s/%s/\u0027 % ("},{"line_number":2166,"context_line":"                        quote(seg_container.encode(\u0027utf8\u0027)),"}],"source_content_type":"text/x-python","patch_set":12,"id":"7f96bb07_11c6f349","line":2163,"updated":"2018-01-13 00:29:30.000000000","message":"Oh, I guess we pull it out here...\n\nI might be a little happier with a pop() or something, to limit our API exposure -- it feels like an internal detail that ends up leaking out to anyone else that might want to consume the service API. I\u0027ll play around with it in a follow-up ...","commit_id":"7a5422716d1b7581cb949014d4a33ea28476f1bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"90e6557fe671cf3a4ddfa979a3a186bfb1e63a71","unresolved":false,"context_lines":[{"line_number":2217,"context_line":"                    response \u003d self._upload_slo_manifest("},{"line_number":2218,"context_line":"                        conn, results, container, obj, put_headers)"},{"line_number":2219,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d response"},{"line_number":2220,"context_line":"                    new_slo_manifest_paths \u003d ["},{"line_number":2221,"context_line":"                        r[\u0027segment_location\u0027] for r in results]"},{"line_number":2222,"context_line":"                else:"},{"line_number":2223,"context_line":"                    res[\u0027response_dict\u0027] \u003d ret"}],"source_content_type":"text/x-python","patch_set":12,"id":"7f96bb07_915c437f","line":2220,"range":{"start_line":2220,"start_character":45,"end_line":2220,"end_character":46},"updated":"2018-01-13 00:29:30.000000000","message":"Might be worth keeping this as a set...","commit_id":"7a5422716d1b7581cb949014d4a33ea28476f1bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4231ab86b6cd821d50df79b5a42b8740ae158218","unresolved":false,"context_lines":[{"line_number":2180,"context_line":"                        response_dict\u003dmr"},{"line_number":2181,"context_line":"                    )"},{"line_number":2182,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2183,"context_line":"            elif \u0027use_slo\u0027 in options and not path and segment_size:"},{"line_number":2184,"context_line":"                segment \u003d 0"},{"line_number":2185,"context_line":"                results \u003d []"},{"line_number":2186,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":13,"id":"7f96bb07_e88a4a19","line":2183,"range":{"start_line":2183,"start_character":51,"end_line":2183,"end_character":67},"updated":"2018-01-15 23:33:04.000000000","message":"This tries to smooth out the upgrade if you were running with default options before. Otherwise, we\u0027d send back some result dict that you may not know how to deal with -- the alternative involved more invasive changes to test_upload_object_job_stream (or else _upload_object_job) than I was quite comfortable with. Although... it\u0027s still weird that this is at all a problem...\n\nHey, do we want\n\n \u0027use_slo\u0027 in options\n\nor do we want\n\n options.get(\u0027use_slo\u0027)\n\n??","commit_id":"edd5d419b2ecd3329d29bbbbfead50244d22708a"},{"author":{"_account_id":26946,"name":"Joel Wright","username":"joeljwright","inactive":true},"change_message_id":"6040e8130e6a8bb26872928a4e190fdf6c05792a","unresolved":false,"context_lines":[{"line_number":2180,"context_line":"                        response_dict\u003dmr"},{"line_number":2181,"context_line":"                    )"},{"line_number":2182,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d mr"},{"line_number":2183,"context_line":"            elif \u0027use_slo\u0027 in options and not path and segment_size:"},{"line_number":2184,"context_line":"                segment \u003d 0"},{"line_number":2185,"context_line":"                results \u003d []"},{"line_number":2186,"context_line":"                while True:"}],"source_content_type":"text/x-python","patch_set":13,"id":"7f96bb07_4652ff3c","line":2183,"range":{"start_line":2183,"start_character":51,"end_line":2183,"end_character":67},"in_reply_to":"7f96bb07_e88a4a19","updated":"2018-01-17 22:33:17.000000000","message":"\u0027use_slo\u0027 will always be in options, so we probably want the latter.","commit_id":"edd5d419b2ecd3329d29bbbbfead50244d22708a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4231ab86b6cd821d50df79b5a42b8740ae158218","unresolved":false,"context_lines":[{"line_number":2221,"context_line":"                    res[\u0027manifest_response_dict\u0027] \u003d response"},{"line_number":2222,"context_line":"                    new_slo_manifest_paths \u003d {"},{"line_number":2223,"context_line":"                        r[\u0027segment_location\u0027] for r in results}"},{"line_number":2224,"context_line":"                    res[\u0027large_object\u0027] \u003d True"},{"line_number":2225,"context_line":"                else:"},{"line_number":2226,"context_line":"                    res[\u0027response_dict\u0027] \u003d ret"},{"line_number":2227,"context_line":"                    res[\u0027large_object\u0027] \u003d False"}],"source_content_type":"text/x-python","patch_set":13,"id":"7f96bb07_28f112ce","line":2224,"updated":"2018-01-15 23:33:04.000000000","message":"Noticed we weren\u0027t setting this in the new mode.","commit_id":"edd5d419b2ecd3329d29bbbbfead50244d22708a"}],"swiftclient/shell.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0588d88572833c8ff3245e37b8b2851017654197","unresolved":false,"context_lines":[{"line_number":1047,"context_line":"        if not options[\u0027use_slo\u0027]:"},{"line_number":1048,"context_line":"            options[\u0027use_slo\u0027] \u003d True"},{"line_number":1049,"context_line":"        if not options[\u0027segment_size\u0027]:"},{"line_number":1050,"context_line":"            options[\u0027segment_size\u0027] \u003d DEFAULT_STDIN_SEGMENT"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"    options[\u0027object_uu_threads\u0027] \u003d options[\u0027object_threads\u0027]"},{"line_number":1053,"context_line":"    with SwiftService(options\u003doptions) as swift:"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f201791_eda4eea7","line":1050,"updated":"2017-06-28 22:10:40.000000000","message":"this is nice - i sorta wish regular chunked object upload worked like this!","commit_id":"270e3b83c971370bf526d281acf347a3fbb212cc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"919912148e236a51dd7f94ebd9176afd15abb664","unresolved":false,"context_lines":[{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    if from_stdin:"},{"line_number":1059,"context_line":"        if not options[\u0027use_slo\u0027]:"},{"line_number":1060,"context_line":"            options[\u0027use_slo\u0027] \u003d True"},{"line_number":1061,"context_line":"        if not options[\u0027segment_size\u0027]:"},{"line_number":1062,"context_line":"            options[\u0027segment_size\u0027] \u003d DEFAULT_STDIN_SEGMENT"},{"line_number":1063,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_ca542b24","line":1060,"updated":"2017-07-25 01:10:19.000000000","message":"Interesting... so we can\u0027t opt out of the SLO-ness?\n\nWhat happens when the user only has write access to the one container? I guess they\u0027ll need to include a --segment-container option... how obvious will that be from the error message?","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"190013da1ab4acee78198432c727369b136e3498","unresolved":false,"context_lines":[{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    if from_stdin:"},{"line_number":1059,"context_line":"        if not options[\u0027use_slo\u0027]:"},{"line_number":1060,"context_line":"            options[\u0027use_slo\u0027] \u003d True"},{"line_number":1061,"context_line":"        if not options[\u0027segment_size\u0027]:"},{"line_number":1062,"context_line":"            options[\u0027segment_size\u0027] \u003d DEFAULT_STDIN_SEGMENT"},{"line_number":1063,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_9b6413e6","line":1060,"in_reply_to":"ff346bd7_ca542b24","updated":"2017-07-25 21:59:09.000000000","message":"I guess it\u0027s not terrible:\n\n $ dd if\u003d/dev/zero | swift upload c - --object-name foo\n Warning: failed to create container \u0027c\u0027: 403 Forbidden: \u003chtml\u003e\u003ch1\u003eForbidden\u003c/h1\u003e\u003cp\u003eAccess was denied to this resourc\n Warning: failed to create container \u0027c_segments\u0027: 403 Forbidden: \u003chtml\u003e\u003ch1\u003eForbidden\u003c/h1\u003e\u003cp\u003eAccess was denied to this resourc\n Object PUT failed: http://saio:8080/v1/AUTH_test/c_segments/foo/slo/1501018432.000000/10485760/00000000 403 Forbidden  [first 60 chars of response] \u003chtml\u003e\u003ch1\u003eForbidden\u003c/h1\u003e\u003cp\u003eAccess was denied to this resourc\n\nThe solution still isn\u0027t obvious, though -- particularly when the object is in the 11MB-5GB range where it totally *could* be a single object.\n\nIt\u0027s also a little annoying that even when everything fits in a single buffer, we still try to create the segments container.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"27dd5f6853f9f2adb43615c597c6a81df3ea9315","unresolved":false,"context_lines":[{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    if from_stdin:"},{"line_number":1059,"context_line":"        if not options[\u0027use_slo\u0027]:"},{"line_number":1060,"context_line":"            options[\u0027use_slo\u0027] \u003d True"},{"line_number":1061,"context_line":"        if not options[\u0027segment_size\u0027]:"},{"line_number":1062,"context_line":"            options[\u0027segment_size\u0027] \u003d DEFAULT_STDIN_SEGMENT"},{"line_number":1063,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"ff346bd7_3599b61e","line":1060,"in_reply_to":"ff346bd7_ca542b24","updated":"2017-07-25 23:35:35.000000000","message":"Right, I\u0027m less concerned about the access issue, but I agree that it\u0027s not great that there is no way to say \"upload as a single object from the standard input stream\". I think we have to introduce another option to enable that.","commit_id":"0549f7b60baae8788e9d37020e11e85ae7b2961c"}],"tests/unit/test_service.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"55f26d188936589b9c43447b90789e5ac5e1a27c","unresolved":false,"context_lines":[{"line_number":1110,"context_line":"        for resp in responses:"},{"line_number":1111,"context_line":"            self.assertFalse(\u0027error\u0027 in resp)"},{"line_number":1112,"context_line":"            self.assertTrue(resp[\u0027success\u0027])"},{"line_number":1113,"context_line":"        self.assertEqual(6, len(responses))"},{"line_number":1114,"context_line":"        container_resp, segment_container_resp \u003d responses[0:2]"},{"line_number":1115,"context_line":"        segment_response \u003d responses[2:4]"},{"line_number":1116,"context_line":"        upload_obj_resp \u003d responses[-1]"}],"source_content_type":"text/x-python","patch_set":10,"id":"ff82abbf_d7bbcde2","line":1113,"updated":"2017-11-20 20:50:43.000000000","message":"Yeah, I don\u0027t understand what Timur is trying to accomplish here. He verifies that 0, 1 are create_container, then 2,3 are segments (we have 2 of 1024), and then 4 or -1 is the response. I guess there\u0027s sometimes a zero-length that squeezes at the end?\n\nI\u0027m going to re-push with length 5, but leave [-1] in as Timur wrote it as a reminder.","commit_id":"63992a29071cbefd4dbb5d5a85b3357bf9e07068"}]}