)]}'
{"fluentd/templates/etc/_td-agent.conf.tpl":[{"author":{"_account_id":11636,"name":"Mateusz Blaszkowski","email":"mateusz.blaszkowski@intel.com","username":"mateuszb"},"change_message_id":"aafce2dbba11536af343112fea59423b785fd5b4","unresolved":false,"context_lines":[{"line_number":116,"context_line":"  path /var/log/containers/*.log"},{"line_number":117,"context_line":"  pos_file fluentd-docker.pos"},{"line_number":118,"context_line":"  time_format %Y-%m-%dT%H:%M:%S"},{"line_number":119,"context_line":"  tag openstack.*"},{"line_number":120,"context_line":"  format json"},{"line_number":121,"context_line":"  read_from_head true"},{"line_number":122,"context_line":"\u003c/source\u003e"}],"source_content_type":"text/x-smarty","patch_set":1,"id":"df3967d1_b23d902c","line":119,"updated":"2017-08-03 05:55:00.000000000","message":"Shouldn\u0027t it be \u0027tag kubernetes.*\u0027?","commit_id":"6ea4e509dc55ccebcd0ea807b2f03727308df590"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1592cf1951ea905255fe560641d341005de3b59f","unresolved":false,"context_lines":[{"line_number":121,"context_line":"  read_from_head true"},{"line_number":122,"context_line":"\u003c/source\u003e"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"# TODO: How to join multiline Python stacktraces into a single entry?"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"# This adds kubernetes_metadata to all logs."},{"line_number":127,"context_line":"\u003cfilter **\u003e"}],"source_content_type":"text/x-smarty","patch_set":1,"id":"df3967d1_5d0c27eb","line":124,"range":{"start_line":124,"start_character":0,"end_line":124,"end_character":69},"updated":"2017-08-03 13:57:32.000000000","message":"This likely could be addressed with regex\u0027s. it might not be elaborate initially, but heres a blog post explaining such an approach: http://work.haufegroup.io/fluentd-log-parsing/","commit_id":"6ea4e509dc55ccebcd0ea807b2f03727308df590"},{"author":{"_account_id":11636,"name":"Mateusz Blaszkowski","email":"mateusz.blaszkowski@intel.com","username":"mateuszb"},"change_message_id":"aafce2dbba11536af343112fea59423b785fd5b4","unresolved":false,"context_lines":[{"line_number":144,"context_line":"  regexp1 container_name ^(barbican-api|cinder-api|cinder-scheduler|cinder-volume|cinder-backup|glance-api|glance-registry|heat-api|geat-cfn|heat-cloudwatch|heat-engine|horizon|keystone-api|magnum-api|magnum-conductor|mistral-api|mistral-executor|mistral-engine|neutron-dhcp-agent|neutron-l3-agent|ovs-agent|neutron-server|nova-osapi|nova-api|nova-conductor|nova-consoleauth|nova-novncproxy|nova-scheduler|senlin-api|senlin-engine)$"},{"line_number":145,"context_line":"\u003c/filter\u003e"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"# Here we\u0027re excluding non-OpenStack services from openstack.*"},{"line_number":148,"context_line":"\u003cfilter kubernetes.**\u003e"},{"line_number":149,"context_line":"  @type grep"},{"line_number":150,"context_line":"  # TODO: Maybe this can be generated somehow?"}],"source_content_type":"text/x-smarty","patch_set":1,"id":"df3967d1_32c520f2","line":147,"updated":"2017-08-03 05:55:00.000000000","message":"excluding OpenStack services from kubernetes.*","commit_id":"6ea4e509dc55ccebcd0ea807b2f03727308df590"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"eef007867102332dd391ae23a31a9b9645c846c9","unresolved":false,"context_lines":[{"line_number":138,"context_line":"# Here we\u0027re excluding non-OpenStack services from openstack.*"},{"line_number":139,"context_line":"\u003cfilter openstack.**\u003e"},{"line_number":140,"context_line":"  @type grep"},{"line_number":141,"context_line":"  # TODO(dulek): Are we able to generate this list?"},{"line_number":142,"context_line":"  regexp1 container_name ^(barbican-api|cinder-api|cinder-scheduler|cinder-volume|cinder-backup|glance-api|glance-registry|heat-api|geat-cfn|heat-cloudwatch|heat-engine|horizon|keystone-api|magnum-api|magnum-conductor|mistral-api|mistral-executor|mistral-engine|neutron-dhcp-agent|neutron-l3-agent|ovs-agent|neutron-server|nova-osapi|nova-api|nova-conductor|nova-consoleauth|nova-novncproxy|nova-scheduler|senlin-api|senlin-engine)$"},{"line_number":143,"context_line":"\u003c/filter\u003e"},{"line_number":144,"context_line":""}],"source_content_type":"text/x-smarty","patch_set":2,"id":"9f436f4f_c4bd7269","line":141,"range":{"start_line":141,"start_character":4,"end_line":141,"end_character":51},"updated":"2017-08-08 14:02:38.000000000","message":"I\u0027ve been back and forth on this point. We could generate this through the values file somehow (a section for enabled openstack services fed into a helm-toolkit function to return the pattern?), but how would we handle services originally disabled being added to the node later? Perhaps they all default to on, and a disclaimer for disabling them? That\u0027s my initial thought anyway","commit_id":"4d47e2fb89014be652a815e7950509827c46a8fd"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"e1e8046afdee26964490b2fc59cdb3e23dd0a07e","unresolved":false,"context_lines":[{"line_number":138,"context_line":"# Here we\u0027re excluding non-OpenStack services from openstack.*"},{"line_number":139,"context_line":"\u003cfilter openstack.**\u003e"},{"line_number":140,"context_line":"  @type grep"},{"line_number":141,"context_line":"  # TODO(dulek): Are we able to generate this list?"},{"line_number":142,"context_line":"  regexp1 container_name ^(barbican-api|cinder-api|cinder-scheduler|cinder-volume|cinder-backup|glance-api|glance-registry|heat-api|geat-cfn|heat-cloudwatch|heat-engine|horizon|keystone-api|magnum-api|magnum-conductor|mistral-api|mistral-executor|mistral-engine|neutron-dhcp-agent|neutron-l3-agent|ovs-agent|neutron-server|nova-osapi|nova-api|nova-conductor|nova-consoleauth|nova-novncproxy|nova-scheduler|senlin-api|senlin-engine)$"},{"line_number":143,"context_line":"\u003c/filter\u003e"},{"line_number":144,"context_line":""}],"source_content_type":"text/x-smarty","patch_set":2,"id":"9f436f4f_c68556af","line":141,"range":{"start_line":141,"start_character":4,"end_line":141,"end_character":51},"in_reply_to":"9f436f4f_c4bd7269","updated":"2017-08-09 13:46:23.000000000","message":"I\u0027ve thought of a few ways here, but nothing really fits. Basic assumption here is that we should have only oslo.log logs in openstack.**, as those will be then decomposed in line 154.\n\n1. We could filter on kubernetes namespace, but \"openstack\" includes also mariadb, rabbitmq, etcd, memcached.\n2. We could filter those services out, but we\u0027re left with bootstrap, db init, db sync jobs, as well as other jobs init containers.\n3. We could modify oslo.log\u0027s format to make it easily grepable, but this sound like an overkill - what if someone wants to use kubectl logs to see logs?\n\nWell, making it configurable in Values.yaml seems like a pretty good start to make this cleaner, so I\u0027ll begin with that.","commit_id":"4d47e2fb89014be652a815e7950509827c46a8fd"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"eef007867102332dd391ae23a31a9b9645c846c9","unresolved":false,"context_lines":[{"line_number":160,"context_line":"  reserve_data true"},{"line_number":161,"context_line":"\u003c/filter\u003e"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"# TODO(dulek): To concatenate multiline Python tracebacks from OpenStack"},{"line_number":164,"context_line":"#              services we would need fluent-plugin-concat installed. To be"},{"line_number":165,"context_line":"#              addressed when we\u0027ll start to use a custom fluentd image."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"# Example:"},{"line_number":168,"context_line":"# I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]"}],"source_content_type":"text/x-smarty","patch_set":2,"id":"9f436f4f_445f4241","line":165,"range":{"start_line":163,"start_character":15,"end_line":165,"end_character":72},"updated":"2017-08-08 14:02:38.000000000","message":"If you\u0027re up to the task, we should include such a custom image as part of this effort. The ability to get multiline tracebacks is high priority for fluentd going forward, at least with OpenStack services to start.","commit_id":"4d47e2fb89014be652a815e7950509827c46a8fd"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"e1e8046afdee26964490b2fc59cdb3e23dd0a07e","unresolved":false,"context_lines":[{"line_number":160,"context_line":"  reserve_data true"},{"line_number":161,"context_line":"\u003c/filter\u003e"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"# TODO(dulek): To concatenate multiline Python tracebacks from OpenStack"},{"line_number":164,"context_line":"#              services we would need fluent-plugin-concat installed. To be"},{"line_number":165,"context_line":"#              addressed when we\u0027ll start to use a custom fluentd image."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"# Example:"},{"line_number":168,"context_line":"# I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]"}],"source_content_type":"text/x-smarty","patch_set":2,"id":"9f436f4f_266d4204","line":165,"range":{"start_line":163,"start_character":15,"end_line":165,"end_character":72},"in_reply_to":"9f436f4f_445f4241","updated":"2017-08-09 13:46:23.000000000","message":"I wanted to link to IRC discussion, but it looks like a netsplit happened and it\u0027s not logged. Copypasting discussion here:\n\n  \u003cdulek\u003e Guys, any thoughts on where to put any Dockerfiles+code of container we would want to build ourselves?\n  \u003cdulek\u003e I\u0027m asking because in https://review.openstack.org/#/c/490036 I\u0027ve realized that I\u0027m unable to continue without installing custom plugins to fluentd image.\n  \u003cdulek\u003e So the easiest path would be to create our own fluentd image.\n  \u003cportdirect\u003e hmm - off the top of my head  creating a `./tools/images/` dir in addons would sound ok ?\n  \u003cportdirect\u003e the same as we did for osh when we needed a ceph config helper container?\n  \u003cdulek\u003e portdirect: That was my initial thought, but it\u0027s a precedence so I wanted to ask for opinions.\n  \u003cdulek\u003e And for the code of openstack exporter for prometheus? I would follow a similar path, but include the code as well.\n  \u003copenstackgerrit\u003e dave kormann proposed openstack/openstack-helm master: Ceph OSD disk targeting  https://review.openstack.org/457754\n  \u003cportdirect\u003e dulek: I\u0027d house that in the same place - though if theres a lot of code we should proabably find a better home long term\n  \u003cdulek\u003e portdirect: Well, splitting it out of openstack/openstack-helm-addons repo seems pretty easy, so I guess we can start there.\n  \u003cportdirect\u003e sounds good - nice one dulek\n  \u003cv1k0d3n\u003e if we have any images, would it make sense to push those all to a common registry?\n  \u003cv1k0d3n\u003e portdirect, i think you\u0027re sitting on a bunch of docker hub orgs named various things similar to openstack-helm, is that right?\n  \u003cv1k0d3n\u003e could one of those be used for the general team?\n  \u003cv1k0d3n\u003e (this is to dulek\u0027s point of having custom images)\n  \u003cdulek\u003e v1k0d3n: Yeah, I\u0027ve also thought of that.\n  \u003cdulek\u003e My only issue here is that I don\u0027t think we want to become kolla-like project…\n  \u003cv1k0d3n\u003e oh i totally agree.\n  \u003cdulek\u003e From the other hand - being unable to do stuff because there are no images that suit us… That sucks as well.\n  \u003cv1k0d3n\u003e we already have custom images that we\u0027re maintaining...and i believe they are scattered in a couple of places atm.\n  \u003cv1k0d3n\u003e making it even worse imo\n  \u003cdulek\u003e v1k0d3n: Agreed.\n  \u003cv1k0d3n\u003e i\u0027m not exactly sure if the att folks started using docker as a calculated and advertised change, or if that was a one off. i believe kolla has a few cores that are able to push to their kolla registry. not sure how this works though. srwilkers any thoughts?\n  \u003cosh-chatbot\u003e \u003crootfs\u003e portdirect: PTAL https://review.openstack.org/#/c/491887/ thanks\n  \u003cv1k0d3n\u003e be nice to have a single repo for custom images, and i know at the start the goal was to limit the images. i think this is why we used ceph upstream in fact...(at least when i was there, this was the argument).\n  \u003cv1k0d3n\u003e i just don\u0027t want to get caught contradicting ourselves.\n  \u003cportdirect\u003e I think attcomdevs quay would make the most sense - this is what the kolla peeps have indicated is their prefered solution.\n  \u003copenstackgerrit\u003e Larry Rensing proposed openstack/openstack-helm-addons master: Gnocchi chart  https://review.openstack.org/472348\n  \u003cportdirect\u003e I\u0027ll make sure any images that are not there (the ceph config one) are moved today - good catch v1k0d3n\n  \u003cv1k0d3n\u003e that makes sense. i can see why they\u0027d prefer that. awesome portdirect.","commit_id":"4d47e2fb89014be652a815e7950509827c46a8fd"}]}