)]}'
{"ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2":[{"author":{"_account_id":30491,"name":"Radosław Piliszek","display_name":"Radek","email":"radek@piliszek.it","username":"yoctozepto","status":"self-employed techologist, collaborating mostly with 7bulls.com"},"change_message_id":"876d21b0364aa3eee316deb5e9900011b290bc79","unresolved":false,"context_lines":[{"line_number":43,"context_line":"    @type rewrite_tag_filter"},{"line_number":44,"context_line":"    rewriterule1 Payload ^\\d{6} infra.mariadb.mysqld_safe"},{"line_number":45,"context_line":"    rewriterule2 Payload ^\\d{4}-\\d{2}-\\d{2} infra.mariadb.mysqld"},{"line_number":46,"context_line":"\u003c/match\u003e"}],"source_content_type":"text/x-jinja2","patch_set":5,"id":"3fa7e38b_2ac48291","line":46,"updated":"2019-11-20 19:59:46.000000000","message":"side note: do we still have 0.12 anywhere?","commit_id":"233a187a76c69e4c892a4ad70e710ee67836dfdd"}],"ansible/roles/common/templates/conf/filter/02-parser.conf.j2":[{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"430e24d3f1fa9cae7907ae62c343fa13fdd6b437","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Parse MariaDB logs with 6 digit date format (mysqld_safe)"},{"line_number":2,"context_line":"\u003cfilter infra.mariadb.mysqld_safe\u003e"},{"line_number":3,"context_line":"    @type parser"},{"line_number":4,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{6} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e(\\[(?\u003clog_level\u003e\\w+)\\]|mysqld_safe) +.*)/"},{"line_number":5,"context_line":"    time_format %y%m%d %k:%M:%S"},{"line_number":6,"context_line":"    time_key Timestamp"},{"line_number":7,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_2df591cb","line":4,"range":{"start_line":4,"start_character":79,"end_line":4,"end_character":89},"updated":"2019-10-29 11:39:38.000000000","message":"Does this one even emit a log level?","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":30438,"name":"Isaac Prior","email":"isaac@stackhpc.com","username":"Wasaac"},"change_message_id":"be52451229b2c485f5a33d2060cd107be2b9a676","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Parse MariaDB logs with 6 digit date format (mysqld_safe)"},{"line_number":2,"context_line":"\u003cfilter infra.mariadb.mysqld_safe\u003e"},{"line_number":3,"context_line":"    @type parser"},{"line_number":4,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{6} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e(\\[(?\u003clog_level\u003e\\w+)\\]|mysqld_safe) +.*)/"},{"line_number":5,"context_line":"    time_format %y%m%d %k:%M:%S"},{"line_number":6,"context_line":"    time_key Timestamp"},{"line_number":7,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_adf4215c","line":4,"range":{"start_line":4,"start_character":79,"end_line":4,"end_character":89},"in_reply_to":"3fa7e38b_2df591cb","updated":"2019-10-29 14:20:12.000000000","message":"I\u0027ve not seen a mysqld_safe message with a log level here. This is left over from the regex originally used in `input/02-mariadb.conf.j2` where it caught both mysqld_safe and mysqld messages.\nWhether it should be removed for clarity or left to catch mysqld messages still using the old format (just in case) is fine with me either way.","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"c121f402f1433ca7dda635436462831c1928a8db","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# Parse MariaDB logs with 6 digit date format (mysqld_safe)"},{"line_number":2,"context_line":"\u003cfilter infra.mariadb.mysqld_safe\u003e"},{"line_number":3,"context_line":"    @type parser"},{"line_number":4,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{6} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e(\\[(?\u003clog_level\u003e\\w+)\\]|mysqld_safe) +.*)/"},{"line_number":5,"context_line":"    time_format %y%m%d %k:%M:%S"},{"line_number":6,"context_line":"    time_key Timestamp"},{"line_number":7,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_c2f5a752","line":4,"range":{"start_line":4,"start_character":79,"end_line":4,"end_character":89},"in_reply_to":"3fa7e38b_adf4215c","updated":"2019-10-29 14:29:18.000000000","message":"+1 to remove it to avoid confusing people","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"430e24d3f1fa9cae7907ae62c343fa13fdd6b437","unresolved":false,"context_lines":[{"line_number":11,"context_line":"# Parse MariaDB logs with 8 digit date format (mysqld)"},{"line_number":12,"context_line":"\u003cfilter infra.mariadb.mysqld\u003e"},{"line_number":13,"context_line":"    @type parser"},{"line_number":14,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{4}-\\d{2}-\\d{2} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e\\w+ +(\\[(?\u003clog_level\u003e\\w+)\\]|\\w+: +(?\u003clog_level\u003e\\w+):).*)/"},{"line_number":15,"context_line":"    time_format %Y-%m-%d %k:%M:%S"},{"line_number":16,"context_line":"    time_key Timestamp"},{"line_number":17,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_6d0f09e0","line":14,"range":{"start_line":14,"start_character":122,"end_line":14,"end_character":133},"updated":"2019-10-29 11:39:38.000000000","message":"Are there any examples of this second capture? They all seem to be in square brackets.","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"430e24d3f1fa9cae7907ae62c343fa13fdd6b437","unresolved":false,"context_lines":[{"line_number":11,"context_line":"# Parse MariaDB logs with 8 digit date format (mysqld)"},{"line_number":12,"context_line":"\u003cfilter infra.mariadb.mysqld\u003e"},{"line_number":13,"context_line":"    @type parser"},{"line_number":14,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{4}-\\d{2}-\\d{2} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e\\w+ +(\\[(?\u003clog_level\u003e\\w+)\\]|\\w+: +(?\u003clog_level\u003e\\w+):).*)/"},{"line_number":15,"context_line":"    time_format %Y-%m-%d %k:%M:%S"},{"line_number":16,"context_line":"    time_key Timestamp"},{"line_number":17,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_2d703172","line":14,"range":{"start_line":14,"start_character":94,"end_line":14,"end_character":136},"updated":"2019-10-29 11:39:38.000000000","message":"I\u0027m not seeing the log_level in Elasticsearch","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":30438,"name":"Isaac Prior","email":"isaac@stackhpc.com","username":"Wasaac"},"change_message_id":"be52451229b2c485f5a33d2060cd107be2b9a676","unresolved":false,"context_lines":[{"line_number":11,"context_line":"# Parse MariaDB logs with 8 digit date format (mysqld)"},{"line_number":12,"context_line":"\u003cfilter infra.mariadb.mysqld\u003e"},{"line_number":13,"context_line":"    @type parser"},{"line_number":14,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{4}-\\d{2}-\\d{2} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e\\w+ +(\\[(?\u003clog_level\u003e\\w+)\\]|\\w+: +(?\u003clog_level\u003e\\w+):).*)/"},{"line_number":15,"context_line":"    time_format %Y-%m-%d %k:%M:%S"},{"line_number":16,"context_line":"    time_key Timestamp"},{"line_number":17,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_ad674125","line":14,"range":{"start_line":14,"start_character":122,"end_line":14,"end_character":133},"in_reply_to":"3fa7e38b_6d0f09e0","updated":"2019-10-29 14:20:12.000000000","message":"One example I found was in the InnodDB output like this:\n\"\"\"\n2018-09-07 12:13:15 7fbde005fb00 InnoDB: Error: Column last_update in table \"mysql\".\"innodb_table_stats\" is INT UNSIGNED NOT NULL but should be BINARY(4) NOT NULL (type mismatch).\n\"\"\"","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"c121f402f1433ca7dda635436462831c1928a8db","unresolved":false,"context_lines":[{"line_number":11,"context_line":"# Parse MariaDB logs with 8 digit date format (mysqld)"},{"line_number":12,"context_line":"\u003cfilter infra.mariadb.mysqld\u003e"},{"line_number":13,"context_line":"    @type parser"},{"line_number":14,"context_line":"    format /^(?\u003cTimestamp\u003e\\d{4}-\\d{2}-\\d{2} {1,2}\\d{1,2}:\\d{1,2}:\\d{1,2}) +(?\u003cPayload\u003e\\w+ +(\\[(?\u003clog_level\u003e\\w+)\\]|\\w+: +(?\u003clog_level\u003e\\w+):).*)/"},{"line_number":15,"context_line":"    time_format %Y-%m-%d %k:%M:%S"},{"line_number":16,"context_line":"    time_key Timestamp"},{"line_number":17,"context_line":"    key_name Payload"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_02d73fc1","line":14,"range":{"start_line":14,"start_character":122,"end_line":14,"end_character":133},"in_reply_to":"3fa7e38b_ad674125","updated":"2019-10-29 14:29:18.000000000","message":"Great, thanks","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"}],"ansible/roles/common/templates/conf/input/02-mariadb.conf.j2":[{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"430e24d3f1fa9cae7907ae62c343fa13fdd6b437","unresolved":false,"context_lines":[{"line_number":6,"context_line":"  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos"},{"line_number":7,"context_line":"  tag infra.mariadb"},{"line_number":8,"context_line":"  format multiline"},{"line_number":9,"context_line":"  format_firstline /^\\d{4}/"},{"line_number":10,"context_line":"  format1 /^(?\u003cPayload\u003e.*)/"},{"line_number":11,"context_line":"  enable_watch_timer false"},{"line_number":12,"context_line":"\u003c/source\u003e"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_b2ed5c26","line":9,"range":{"start_line":9,"start_character":19,"end_line":9,"end_character":26},"updated":"2019-10-29 11:39:38.000000000","message":"This seems a bit fragile, although I can\u0027t see any examples of it matching when it shouldn\u0027t in the logs. We could match on one time/date format or the other","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"c121f402f1433ca7dda635436462831c1928a8db","unresolved":false,"context_lines":[{"line_number":6,"context_line":"  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos"},{"line_number":7,"context_line":"  tag infra.mariadb"},{"line_number":8,"context_line":"  format multiline"},{"line_number":9,"context_line":"  format_firstline /^\\d{4}/"},{"line_number":10,"context_line":"  format1 /^(?\u003cPayload\u003e.*)/"},{"line_number":11,"context_line":"  enable_watch_timer false"},{"line_number":12,"context_line":"\u003c/source\u003e"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_e2546356","line":9,"range":{"start_line":9,"start_character":19,"end_line":9,"end_character":26},"in_reply_to":"3fa7e38b_4284570b","updated":"2019-10-29 14:29:18.000000000","message":"Yeah, that\u0027s what I was worried about. I think given that the chance of matching a multi-line log with the current regex isn\u0027t vanishingly small, I\u0027d rather see a regex that at least matched the date as you suggest.","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"7b2bb69a5015f33c0b6dc91baa8cf1e90b6786fb","unresolved":false,"context_lines":[{"line_number":6,"context_line":"  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos"},{"line_number":7,"context_line":"  tag infra.mariadb"},{"line_number":8,"context_line":"  format multiline"},{"line_number":9,"context_line":"  format_firstline /^\\d{4}/"},{"line_number":10,"context_line":"  format1 /^(?\u003cPayload\u003e.*)/"},{"line_number":11,"context_line":"  enable_watch_timer false"},{"line_number":12,"context_line":"\u003c/source\u003e"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_ed13f9ff","line":9,"range":{"start_line":9,"start_character":19,"end_line":9,"end_character":26},"in_reply_to":"3fa7e38b_b2ed5c26","updated":"2019-10-29 11:40:48.000000000","message":"I\u0027m seeing lots of match warnings in the fluentd container stdout. Eg:\n\n`\n2019-10-29 09:57:25 +0000 [warn]: got incomplete line before first line from /var/log/kolla/mariadb/mariadb.log: \"2019-10-29  9:57:25 139690725649152 [Note] WSREP: applier thread exiting (code:0)\\n\"\n2019-10-29 09:57:25 +0000 [warn]: got incomplete line before first line from /var/log/kolla/mariadb/mariadb.log: \"2019-10-29  9:57:25 139690725341952 [Note] WSREP: applier thread exiting (code:6)\\n\"\n2019-10-29 09:57:25 +0000 [warn]: got incomplete line before first line from /var/log/kolla/mariadb/mariadb.log: \"2019-10-29  9:57:25 139690729777920 [Note] WSREP: applier thread exiting (code:6)\\n\"\n2019-10-29 09:57:25 +0000 [warn]: got incomplete line before first line from /var/log/kolla/mariadb/mariadb.log: \"2019-10-29  9:57:25 139690725034752 [Note] WSREP: applier thread exiting (code:6)\\n\"\n`","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"},{"author":{"_account_id":30438,"name":"Isaac Prior","email":"isaac@stackhpc.com","username":"Wasaac"},"change_message_id":"be52451229b2c485f5a33d2060cd107be2b9a676","unresolved":false,"context_lines":[{"line_number":6,"context_line":"  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos"},{"line_number":7,"context_line":"  tag infra.mariadb"},{"line_number":8,"context_line":"  format multiline"},{"line_number":9,"context_line":"  format_firstline /^\\d{4}/"},{"line_number":10,"context_line":"  format1 /^(?\u003cPayload\u003e.*)/"},{"line_number":11,"context_line":"  enable_watch_timer false"},{"line_number":12,"context_line":"\u003c/source\u003e"}],"source_content_type":"text/x-jinja2","patch_set":3,"id":"3fa7e38b_4284570b","line":9,"range":{"start_line":9,"start_character":19,"end_line":9,"end_character":26},"in_reply_to":"3fa7e38b_b2ed5c26","updated":"2019-10-29 14:20:12.000000000","message":"The lowest common denominator for both log formats is the 4 digit year. Presumably `/^(\\d{6}|\\d{4}-\\d{2}-\\d{2})/` would also work if you were worried about multiline logs starting with 4 digits. \nI\u0027m cautious about adding more regex than necessary for the sake of performance though.","commit_id":"0d78fd04cf0b8af5b96b3ee90c43c8c6214cb31c"}]}