)]}'
{"os_loganalyze/generator.py":[{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"c4b59f0d5a5366918ed93b06bd7cf0d440a57b12","unresolved":false,"context_lines":[{"line_number":198,"context_line":"    if not flines_generator:"},{"line_number":199,"context_line":"        raise NoSuchFile()"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"    return logname, flines_generator, file_headers"}],"source_content_type":"text/x-python","patch_set":6,"id":"baa041b7_654f07be","line":201,"updated":"2015-04-03 16:19:30.000000000","message":"When I took a stab at this I was not super fond of further expanding the inputs and/or outputs of this function. As we make this function responsible for more and more functionality it becomes unwieldy.\n\nThat said my expansion of https://review.openstack.org/#/c/170234/2/os_loganalyze/wsgi.py\u0027s application() to take on some of this functionality is also unwieldy.\n\nI think a saner approach would be to have a class that looks something like:\n\n  class IterableBuffer(object):\n      def __init__(self, environ, root_path, config\u003dNone):\n          # Copy in params to self\n          # get logname and logpath (checking the path)\n          # self.swift_con \u003d None\n      def _getswiftconnection(self):\n          # now is the time to address the above todo I think\n      def get_swift_line_generator(self):\n          # I would like to see this refactored like I refactored it in\n          # https://review.openstack.org/#/c/170234/2/os_loganalyze/generator.py\n          # basically have one function to get obj and ensure it is\n          # of correct type and another to return the Generator\n          # type iterator function. But that can be a followup\n          # change since its unrelated to this functionality.\n      def get(self):\n          # do stuff\n          # return flines_generator\n\nThen callers can get the generator and refer to file_iterator.log(name|path) and file_iterator.headers in a slightly more readable manner.\n\nAlso note that I am fine with making the above suggestion happen in a followup change to reduce churn here.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":7069,"name":"Joshua Hesketh","email":"josh@nitrotech.org","username":"jhesketh"},"change_message_id":"65868685b14657d5f1deaace7c4b46c8de4aa526","unresolved":false,"context_lines":[{"line_number":198,"context_line":"    if not flines_generator:"},{"line_number":199,"context_line":"        raise NoSuchFile()"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"    return logname, flines_generator, file_headers"}],"source_content_type":"text/x-python","patch_set":6,"id":"9aa53dc9_29f8e8e4","line":201,"in_reply_to":"baa041b7_654f07be","updated":"2015-04-07 12:52:37.000000000","message":"Yeah I agree (and had similar thoughts when I first wrote this).\n\nI think this might be easier to do as a follow on change.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"}],"os_loganalyze/tests/test_wsgi.py":[{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"c4b59f0d5a5366918ed93b06bd7cf0d440a57b12","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        # Test the passthrough filter returns a tarball stream"},{"line_number":77,"context_line":"        gen \u003d self.get_generator(\u0027openstack_logo.png\u0027)"},{"line_number":78,"context_line":"        first \u003d gen.next()"},{"line_number":79,"context_line":"        self.assertNotIn(\u0027html\u0027, first)"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"class KnownFilesMixin(object):"}],"source_content_type":"text/x-python","patch_set":6,"id":"baa041b7_c57ff364","line":79,"updated":"2015-04-03 16:19:30.000000000","message":"Can we add a slightly more robust assertion here by reading in the file on disk in this test, then asserting file[:len(first)] is equal to first?","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":7069,"name":"Joshua Hesketh","email":"josh@nitrotech.org","username":"jhesketh"},"change_message_id":"65868685b14657d5f1deaace7c4b46c8de4aa526","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        # Test the passthrough filter returns a tarball stream"},{"line_number":77,"context_line":"        gen \u003d self.get_generator(\u0027openstack_logo.png\u0027)"},{"line_number":78,"context_line":"        first \u003d gen.next()"},{"line_number":79,"context_line":"        self.assertNotIn(\u0027html\u0027, first)"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"class KnownFilesMixin(object):"}],"source_content_type":"text/x-python","patch_set":6,"id":"9aa53dc9_0921a44d","line":79,"in_reply_to":"baa041b7_c57ff364","updated":"2015-04-07 12:52:37.000000000","message":"Done","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"}],"os_loganalyze/util.py":[{"author":{"_account_id":6786,"name":"Sergey Lukjanov","email":"me@slukjanov.name","username":"slukjanov"},"change_message_id":"ddb2f75486f24c85bcf2c006f94fb3046ddcd809","unresolved":false,"context_lines":[{"line_number":55,"context_line":"    resp[\u0027accept-ranges\u0027] \u003d \u0027bytes\u0027"},{"line_number":56,"context_line":"    resp[\u0027last-modified\u0027] \u003d time.strftime("},{"line_number":57,"context_line":"        \"%a, %d %b %Y %H:%M:%S GMT\", time.gmtime(os.path.getmtime(file_path)))"},{"line_number":58,"context_line":"    resp[\u0027etag\u0027] \u003d \u0027\u0027"},{"line_number":59,"context_line":"    resp[\u0027x-trans-id\u0027] \u003d str(time.time())"},{"line_number":60,"context_line":"    resp[\u0027date\u0027] \u003d time.strftime(\"%a, %d %b %Y %H:%M:%S GMT\")"},{"line_number":61,"context_line":"    resp[\u0027content-type\u0027] \u003d get_file_mime(file_path)"}],"source_content_type":"text/x-python","patch_set":6,"id":"3a961159_bee97781","line":58,"updated":"2015-01-19 17:58:06.000000000","message":"I\u0027m not sure that it\u0027s a good idea to set e-tag to the empty string. We could use hash as etag.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":6786,"name":"Sergey Lukjanov","email":"me@slukjanov.name","username":"slukjanov"},"change_message_id":"79678e85afdc3ddb8980a0070e0fdcac52d15e01","unresolved":false,"context_lines":[{"line_number":55,"context_line":"    resp[\u0027accept-ranges\u0027] \u003d \u0027bytes\u0027"},{"line_number":56,"context_line":"    resp[\u0027last-modified\u0027] \u003d time.strftime("},{"line_number":57,"context_line":"        \"%a, %d %b %Y %H:%M:%S GMT\", time.gmtime(os.path.getmtime(file_path)))"},{"line_number":58,"context_line":"    resp[\u0027etag\u0027] \u003d \u0027\u0027"},{"line_number":59,"context_line":"    resp[\u0027x-trans-id\u0027] \u003d str(time.time())"},{"line_number":60,"context_line":"    resp[\u0027date\u0027] \u003d time.strftime(\"%a, %d %b %Y %H:%M:%S GMT\")"},{"line_number":61,"context_line":"    resp[\u0027content-type\u0027] \u003d get_file_mime(file_path)"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba7be1f8_2ddb99ee","line":58,"in_reply_to":"1a930d6b_4090b2ec","updated":"2015-02-20 17:22:04.000000000","message":"ok","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":7069,"name":"Joshua Hesketh","email":"josh@nitrotech.org","username":"jhesketh"},"change_message_id":"02ff94c14f96301a0a01523a1934198b434fdbf3","unresolved":false,"context_lines":[{"line_number":55,"context_line":"    resp[\u0027accept-ranges\u0027] \u003d \u0027bytes\u0027"},{"line_number":56,"context_line":"    resp[\u0027last-modified\u0027] \u003d time.strftime("},{"line_number":57,"context_line":"        \"%a, %d %b %Y %H:%M:%S GMT\", time.gmtime(os.path.getmtime(file_path)))"},{"line_number":58,"context_line":"    resp[\u0027etag\u0027] \u003d \u0027\u0027"},{"line_number":59,"context_line":"    resp[\u0027x-trans-id\u0027] \u003d str(time.time())"},{"line_number":60,"context_line":"    resp[\u0027date\u0027] \u003d time.strftime(\"%a, %d %b %Y %H:%M:%S GMT\")"},{"line_number":61,"context_line":"    resp[\u0027content-type\u0027] \u003d get_file_mime(file_path)"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a930d6b_4090b2ec","line":58,"in_reply_to":"3a961159_bee97781","updated":"2015-01-21 09:44:16.000000000","message":"This could be costly to calculate and seemingly not useful.\n\nWe\u0027re only using the headers here to determine if we should render or download the requested file. It might be useful in the future but I think we should leave it out for now.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"}],"os_loganalyze/wsgi.py":[{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"c4b59f0d5a5366918ed93b06bd7cf0d440a57b12","unresolved":false,"context_lines":[{"line_number":112,"context_line":"        start_response(status, response_headers)"},{"line_number":113,"context_line":"        return [\u0027File Not Found\u0027]"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if use_passthrough_view(file_headers):"},{"line_number":116,"context_line":"        generator \u003d osview.PassthroughView(flines_generator,"},{"line_number":117,"context_line":"                                           file_headers[\u0027content-type\u0027])"},{"line_number":118,"context_line":"    else:"}],"source_content_type":"text/x-python","patch_set":6,"id":"baa041b7_403071e8","line":115,"updated":"2015-04-03 16:19:30.000000000","message":"One of the things I noticed when working on https://review.openstack.org/#/c/170234 is that the apache vhost is set up to only serve .txt.gz and console.html(.gz) files through os-loganalyze.\n\nI think we should either respect that choice here, or possibly have os-loganalyze support to root paths. /htmlify for when we want to get the logs processed by os-loganalyze and /passthrough for when we just need to make the trip to swift.\n\nWhat do you think about that?","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"4f0b1fe23e209de4e0fc906e014f698572e10a0a","unresolved":false,"context_lines":[{"line_number":112,"context_line":"        start_response(status, response_headers)"},{"line_number":113,"context_line":"        return [\u0027File Not Found\u0027]"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if use_passthrough_view(file_headers):"},{"line_number":116,"context_line":"        generator \u003d osview.PassthroughView(flines_generator,"},{"line_number":117,"context_line":"                                           file_headers[\u0027content-type\u0027])"},{"line_number":118,"context_line":"    else:"}],"source_content_type":"text/x-python","patch_set":6,"id":"baa041b7_c0b361e8","line":115,"in_reply_to":"baa041b7_403071e8","updated":"2015-04-03 16:25:46.000000000","message":"Slight correction to my comment here. The apache vhost is only set up to serve files that match that pattern if they are on disk. If the file is found on disk not matching that pattern apache serves it directly. If that file is not found on disk then it is always passed to os-loganalyze in an attempt to serve it from swift.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"c4b59f0d5a5366918ed93b06bd7cf0d440a57b12","unresolved":false,"context_lines":[{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if use_passthrough_view(file_headers):"},{"line_number":116,"context_line":"        generator \u003d osview.PassthroughView(flines_generator,"},{"line_number":117,"context_line":"                                           file_headers[\u0027content-type\u0027])"},{"line_number":118,"context_line":"    else:"},{"line_number":119,"context_line":"        minsev \u003d util.parse_param(environ, \u0027level\u0027, default\u003d\"NONE\")"},{"line_number":120,"context_line":"        limit \u003d util.parse_param(environ, \u0027limit\u0027)"}],"source_content_type":"text/x-python","patch_set":6,"id":"baa041b7_25d1ff57","line":117,"updated":"2015-04-03 16:19:30.000000000","message":"I think this should pass in file_headers as the headers value to PassthroughView instead of passing in only the content-type header. By passing in only the content-type header we lose all of the information retrieved by get_headers_for_file() or given to use by swift.","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":7069,"name":"Joshua Hesketh","email":"josh@nitrotech.org","username":"jhesketh"},"change_message_id":"65868685b14657d5f1deaace7c4b46c8de4aa526","unresolved":false,"context_lines":[{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if use_passthrough_view(file_headers):"},{"line_number":116,"context_line":"        generator \u003d osview.PassthroughView(flines_generator,"},{"line_number":117,"context_line":"                                           file_headers[\u0027content-type\u0027])"},{"line_number":118,"context_line":"    else:"},{"line_number":119,"context_line":"        minsev \u003d util.parse_param(environ, \u0027level\u0027, default\u003d\"NONE\")"},{"line_number":120,"context_line":"        limit \u003d util.parse_param(environ, \u0027limit\u0027)"}],"source_content_type":"text/x-python","patch_set":6,"id":"9aa53dc9_69d5d041","line":117,"in_reply_to":"baa041b7_25d1ff57","updated":"2015-04-07 12:52:37.000000000","message":"Done","commit_id":"757fce83fe288fd8c5ae898547bad70608974e3e"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"72acefb6786adfa32bd70817032493957f3d4c78","unresolved":false,"context_lines":[{"line_number":82,"context_line":"            if os.path.splitext(filename)[1] in [\u0027.txt\u0027, \u0027.html\u0027]:"},{"line_number":83,"context_line":"                return False"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"    return True"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"def application(environ, start_response, root_path\u003dNone,"}],"source_content_type":"text/x-python","patch_set":7,"id":"9aa53dc9_db73704c","line":85,"updated":"2015-04-07 17:18:52.000000000","message":"For this function I think we need to respect the rules at https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project/templates/logs.vhost.erb#n74 because we host text/plain content that isn\u0027t log data or html that we want to process.\n\nOne example of this is javascript uploaded as part of coverage reports, http://logs.openstack.org/82/165682/8/check/nodepool-coverage/08c6340/cover/jquery.min.js\n\nIf you look at the source of that file you will see it got htmlified when it shouldn\u0027t and chrome developer tools says it was content-type: text/plain.\n\nWe can either use the same rules use in the apache vhost above here or have a path called passthrough/ in addition to htmlify/ and the passthrough path would always avoid htmlification. Then apache could use passthrough/ at https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project/templates/logs.vhost.erb#n83","commit_id":"88c20fa46a56b42e98748a21ea8b5c4d5d38356f"}]}
