)]}'
{"specs/osh-lma-stack.rst":[{"author":{"_account_id":17589,"name":"Darla Ahlert","email":"da741q@att.com","username":"darla.ahlert"},"change_message_id":"fc1ea66df744b38a8b410d93862a58a8053aa18d","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":".."},{"line_number":8,"context_line":" This template should be in ReSTructured text. The filename in the git"},{"line_number":9,"context_line":" repository should match the launchpad URL, for example a URL of"},{"line_number":10,"context_line":" https://blueprints.launchpad.net/kolla/+spec/awesome-thing should be named"},{"line_number":11,"context_line":" awesome-thing.rst . Please do not delete any of the sections in this"},{"line_number":12,"context_line":" template. If you have nothing to say for a whole section, just write: None"},{"line_number":13,"context_line":" For help with syntax, see http://sphinx-doc.org/rest.html"},{"line_number":14,"context_line":" To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":17,"context_line":"OSH Logging, Monitoring, and Alerting"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1f1a1f67_ca28b387","line":14,"range":{"start_line":8,"start_character":0,"end_line":14,"end_character":73},"updated":"2017-07-13 19:15:15.000000000","message":"remove","commit_id":"15538251c6b61ee162be9c11ffe7078b47fa6ac8"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"e63b6918e52a86fd6eea1d443bdf705c82f736b3","unresolved":false,"context_lines":[{"line_number":88,"context_line":"2. Kubernetes metrics: pod status, replica availability, job status, etc"},{"line_number":89,"context_line":"3. Ceph metrics: total pool usage, latency, health, etc"},{"line_number":90,"context_line":"4. OpenStack metrics: tenants, networks, flavors, floating IPs, quotas, etc"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"Examples of how these metrics can be used include:"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"1. Add or remove nodes depending on utilization"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1f1a1f67_b7d8177f","line":91,"updated":"2017-07-14 19:28:33.000000000","message":"5. Proactively monitor for stack traces, across the entire deployed infrastructure.","commit_id":"205fea72b5ad4ac40b35bd71093d28811cb90807"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"366b883261bafb3a543e40438bf735674944dd9a","unresolved":false,"context_lines":[{"line_number":88,"context_line":"2. Kubernetes metrics: pod status, replica availability, job status, etc"},{"line_number":89,"context_line":"3. Ceph metrics: total pool usage, latency, health, etc"},{"line_number":90,"context_line":"4. OpenStack metrics: tenants, networks, flavors, floating IPs, quotas, etc"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"Examples of how these metrics can be used include:"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"1. Add or remove nodes depending on utilization"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1f1a1f67_1e1b3fd0","line":91,"in_reply_to":"1f1a1f67_b7d8177f","updated":"2017-07-17 13:08:22.000000000","message":"Done","commit_id":"205fea72b5ad4ac40b35bd71093d28811cb90807"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"2e4d8f0c7f7356679de35148c3ab0d95da342470","unresolved":false,"context_lines":[{"line_number":146,"context_line":"dashboard.  Grafana can be bootstrapped with dashboards for each target scraped,"},{"line_number":147,"context_line":"or the dashboards can be added via Grafana\u0027s web interface directly.  To meet"},{"line_number":148,"context_line":"OpenStack-Helm\u0027s alerting needs, Alertmanager can be used to interface with"},{"line_number":149,"context_line":"Prometheus and send alerts based off Prometheus rule evaluations."},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"The proposal includes the following:"},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f1a1f67_6a1f0db7","line":149,"range":{"start_line":149,"start_character":33,"end_line":149,"end_character":36},"updated":"2017-07-17 14:42:01.000000000","message":"on","commit_id":"411df1e7984a2168b765291547c0dbd6f35b0765"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"ab7584ab4f72e91dbde5f12758e464a6b628c947","unresolved":false,"context_lines":[{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_ea6af317","line":73,"range":{"start_line":73,"start_character":3,"end_line":73,"end_character":8},"updated":"2017-07-17 20:09:13.000000000","message":"Record","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1bbd7cda4ee1126536d0cb702bdaf59e31eba433","unresolved":false,"context_lines":[{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_2aba2b6d","line":73,"range":{"start_line":73,"start_character":3,"end_line":73,"end_character":8},"in_reply_to":"1f1a1f67_ea6af317","updated":"2017-07-17 20:15:11.000000000","message":"Done","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"ab7584ab4f72e91dbde5f12758e464a6b628c947","unresolved":false,"context_lines":[{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_2a5eab75","line":74,"range":{"start_line":74,"start_character":3,"end_line":74,"end_character":8},"updated":"2017-07-17 20:09:13.000000000","message":"Record","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1bbd7cda4ee1126536d0cb702bdaf59e31eba433","unresolved":false,"context_lines":[{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_0ab3ef99","line":74,"range":{"start_line":74,"start_character":3,"end_line":74,"end_character":8},"in_reply_to":"1f1a1f67_2a5eab75","updated":"2017-07-17 20:15:11.000000000","message":"Done","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"ab7584ab4f72e91dbde5f12758e464a6b628c947","unresolved":false,"context_lines":[{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_ca4cd7c9","line":76,"range":{"start_line":76,"start_character":3,"end_line":76,"end_character":8},"updated":"2017-07-17 20:09:13.000000000","message":"Identify","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1bbd7cda4ee1126536d0cb702bdaf59e31eba433","unresolved":false,"context_lines":[{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_caa85721","line":76,"range":{"start_line":76,"start_character":3,"end_line":76,"end_character":8},"in_reply_to":"1f1a1f67_ca4cd7c9","updated":"2017-07-17 20:15:11.000000000","message":"Done","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"ab7584ab4f72e91dbde5f12758e464a6b628c947","unresolved":false,"context_lines":[{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"},{"line_number":80,"context_line":"8. Find all logged events that match a pattern (service, pod, behavior, etc)"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_aa499bb7","line":77,"range":{"start_line":77,"start_character":3,"end_line":77,"end_character":8},"updated":"2017-07-17 20:09:13.000000000","message":"Identify","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1bbd7cda4ee1126536d0cb702bdaf59e31eba433","unresolved":false,"context_lines":[{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"},{"line_number":80,"context_line":"8. Find all logged events that match a pattern (service, pod, behavior, etc)"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_aaa51b49","line":77,"range":{"start_line":77,"start_character":3,"end_line":77,"end_character":8},"in_reply_to":"1f1a1f67_aa499bb7","updated":"2017-07-17 20:15:11.000000000","message":"Done","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":21307,"name":"Brandon B. Jozsa","email":"bjozsa@jinkit.com","username":"v1k0d3n"},"change_message_id":"0d34d1ad0b3866aacbef0e9a2bf66b06796f6815","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Logging Use Cases"},{"line_number":69,"context_line":"-----------------"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"},{"line_number":80,"context_line":"8. Find all logged events that match a pattern (service, pod, behavior, etc)"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Monitoring Use Cases"},{"line_number":83,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_2a858bcf","line":80,"range":{"start_line":71,"start_character":0,"end_line":80,"end_character":76},"updated":"2017-07-17 20:09:22.000000000","message":"\"trace\" could potentially be confused with trace levels (i.e. more granular than debug).","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"1bbd7cda4ee1126536d0cb702bdaf59e31eba433","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Logging Use Cases"},{"line_number":69,"context_line":"-----------------"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Example uses for centralized logging include:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1. Trace compute instance behavior across nodes and services"},{"line_number":74,"context_line":"2. Trace OpenStack service behavior and status"},{"line_number":75,"context_line":"3. Find all backtraces for a tenant id\u0027s uuid"},{"line_number":76,"context_line":"4. Trace issues with infrastructure components, such as RabbitMQ, mariadb, etc"},{"line_number":77,"context_line":"5. Trace issues with Kubernetes components, such as: etcd, CNI, scheduler, etc"},{"line_number":78,"context_line":"6. Organizational auditing needs"},{"line_number":79,"context_line":"7. Visualize history of logged events to determine if event a pattern or outlier"},{"line_number":80,"context_line":"8. Find all logged events that match a pattern (service, pod, behavior, etc)"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Monitoring Use Cases"},{"line_number":83,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f1a1f67_8abfff5f","line":80,"range":{"start_line":71,"start_character":0,"end_line":80,"end_character":76},"in_reply_to":"1f1a1f67_2a858bcf","updated":"2017-07-17 20:15:11.000000000","message":"Done","commit_id":"f96bcd8ec770bacbc66d262896f213697102a74e"},{"author":{"_account_id":8181,"name":"Jaesuk Ahn","email":"bluejay.ahn@gmail.com","username":"blujeay.kr"},"change_message_id":"da7b2fb1e7722164617786279157112e1c6fe55d","unresolved":false,"context_lines":[{"line_number":24,"context_line":"OpenStack-Helm currently lacks a centralized mechanism for providing insight"},{"line_number":25,"context_line":"into the performance of the OpenStack services and infrastructure components."},{"line_number":26,"context_line":"The log formats of the different components in OpenStack-Helm vary, which makes"},{"line_number":27,"context_line":"identifying causes for issues difficult across services.  To support operational"},{"line_number":28,"context_line":"readiness by default, OpenStack-Helm should include components for logging"},{"line_number":29,"context_line":"events in a common format, monitoring metrics at all levels, alerting and alarms"},{"line_number":30,"context_line":"for those metrics, and visualization tools for querying the logs and metrics in"},{"line_number":31,"context_line":"a single pane view."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Platform Requirements"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_dc552b52","line":31,"range":{"start_line":27,"start_character":58,"end_line":31,"end_character":19},"updated":"2017-07-24 15:53:07.000000000","message":"Operational readiness can mean many things. This spec is very good at capturing tooling requirements to monitor OpenStack-Helm environment. However, the most important thing to be operation-ready is insight. what to alert, what these alerts means, what to collect to be able to trigger these alerts. I would like to see a collective effort from the community to make default set of alert list and alert trigger value, default list of metrics to collect, default list of logs to gather. This will be more like knowledge gathering, and can be a separate spec or repository.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":24,"context_line":"OpenStack-Helm currently lacks a centralized mechanism for providing insight"},{"line_number":25,"context_line":"into the performance of the OpenStack services and infrastructure components."},{"line_number":26,"context_line":"The log formats of the different components in OpenStack-Helm vary, which makes"},{"line_number":27,"context_line":"identifying causes for issues difficult across services.  To support operational"},{"line_number":28,"context_line":"readiness by default, OpenStack-Helm should include components for logging"},{"line_number":29,"context_line":"events in a common format, monitoring metrics at all levels, alerting and alarms"},{"line_number":30,"context_line":"for those metrics, and visualization tools for querying the logs and metrics in"},{"line_number":31,"context_line":"a single pane view."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Platform Requirements"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_b9269514","line":31,"range":{"start_line":27,"start_character":58,"end_line":31,"end_character":19},"in_reply_to":"ff346bd7_dc552b52","updated":"2017-07-26 22:32:17.000000000","message":"That is indeed the plan - we need to create a separate specification geared specifically towards what to automatically populate the stack defined in this specification with to be useful and flexible out of the box","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":45,"context_line":"4. Ability to apply custom metadata and uniform format to logs"},{"line_number":46,"context_line":"5. Time-series database for logs collected"},{"line_number":47,"context_line":"6. Backed by highly available storage"},{"line_number":48,"context_line":"7. Configurable log rotation mechanism"},{"line_number":49,"context_line":"8. Ability to perform custom queries against stored logs"},{"line_number":50,"context_line":"9. Single pane visualization capabilities"},{"line_number":51,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_f95e5d74","line":48,"updated":"2017-07-26 22:32:17.000000000","message":"We should likely be specific here - what logs are we committing to manage rotation for? Is it simply what is streamed to us and stored in the various log tools in this stack or are we encompassing actual docker log rotation in this as well?","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":47,"context_line":"6. Backed by highly available storage"},{"line_number":48,"context_line":"7. Configurable log rotation mechanism"},{"line_number":49,"context_line":"8. Ability to perform custom queries against stored logs"},{"line_number":50,"context_line":"9. Single pane visualization capabilities"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Monitoring Requirements"},{"line_number":53,"context_line":"-----------------------"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_d9dcb9e5","line":50,"updated":"2017-07-26 22:32:17.000000000","message":"I would add to this the capability of raising alarms based on these logs - an element that may be missing from our stack is ElastAlert","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":88,"context_line":"2. Kubernetes metrics: pod status, replica availability, job status, etc"},{"line_number":89,"context_line":"3. Ceph metrics: total pool usage, latency, health, etc"},{"line_number":90,"context_line":"4. OpenStack metrics: tenants, networks, flavors, floating IPs, quotas, etc"},{"line_number":91,"context_line":"5. Proactive monitoring of stack traces across all deployed infrastructure"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Examples of how these metrics can be used include:"},{"line_number":94,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_b9b0d58b","line":91,"updated":"2017-07-26 22:32:17.000000000","message":"To this I would also add control plane status - which can be interpreted as a green light for all subcomponents of OpenStack as well as Kubernetes.  To me, this was what Intel was driving at with their health checking and suicide approach but I think it makes more sense to surface it this way.  In other words, if Keystone can not reach rabbitmq, how can we expose that? There are several different ways, but having infrastructure/external reachability/connection tests for each component aggregated can provide the foundations to an eventual openstack-helm \"All is OK or Troubleshoot XYZ\" dashboard","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":96,"context_line":"2. Trigger alerts when desired replicas fall below required number"},{"line_number":97,"context_line":"3. Trigger alerts when services become unavailable or unresponsive"},{"line_number":98,"context_line":"4. Identify etcd performance that could lead to cluster instability"},{"line_number":99,"context_line":"5. Visualize performance to identify trends in traffic or utilization over time"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Proposed Change"},{"line_number":102,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_d926f9ce","line":99,"updated":"2017-07-26 22:32:17.000000000","message":"Also, one of the goals is to be able to surface alarms up to Nagios (or have nagios check for them) - Nagios makes sense for those with existing OSS/BSS integration with nagios but others can tap into these in whatever makes sense for them but this demonstrates we\u0027re able to surface/catch them","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"1f6c87b4f47156fbdf137179286f63c5c2d7b9ae","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"1. Helm chart for Fluentd"},{"line_number":124,"context_line":"2. Helm chart for Elasticsearch"},{"line_number":125,"context_line":"3. Helm chart for Kibana"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"All three charts must include sensible configuration values to make the"},{"line_number":128,"context_line":"logging platform usable by default.  These include: proper input configurations"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_b4f7998b","line":125,"updated":"2017-07-18 18:01:43.000000000","message":"Although I\u0027m not sure whether it\u0027s in MVP territory, I think it\u0027s important to at least leave the door open for log-based alerting, to facilitate alerting when events (or unusual patterns of events) occur.  E.g., some plugin for ES or Kibana that could shoot off an alert based on a periodic query, or, another tool that queries ES and shoots off alerts.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"a0d57f79fdc98fe6f4cb0297cd1e99aea4e37b4c","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"1. Helm chart for Fluentd"},{"line_number":124,"context_line":"2. Helm chart for Elasticsearch"},{"line_number":125,"context_line":"3. Helm chart for Kibana"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"All three charts must include sensible configuration values to make the"},{"line_number":128,"context_line":"logging platform usable by default.  These include: proper input configurations"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_643080e9","line":125,"in_reply_to":"1f1a1f67_7858b92f","updated":"2017-07-26 22:32:17.000000000","message":"ElastAlert for log-based alarming?\n\nWith regard to situations like RabbitMQ/DB - long-term I think this is a /health on every REST API in OpenStack but that is long-term. We had a good session on this at the Summit but we have not followed through on pushing for this in the community.\n\nBarring that, I don\u0027t think log scraping is the only way and in fact I think it won\u0027t be consistent across each service.  That said, an external test from the perspective of the POD (but not the container itself) can likely drive a lot of this - using the config files themselves to find the endpoints to check.  For glance, this may be validating the backend works for storage access.  For keystone, this may be ensuring the backend (LDAP, ...) is reachable.  These things wouldn\u0027t be examining it precisely from the application perspective itself, but it would be very close and it would not result in the termination of PODs as the Liveness approach could have - instead it would bubble up for other actors to make decisions on.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":8181,"name":"Jaesuk Ahn","email":"bluejay.ahn@gmail.com","username":"blujeay.kr"},"change_message_id":"16c1a28eae9a10695b70949e21cd741bb8b29f0e","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"1. Helm chart for Fluentd"},{"line_number":124,"context_line":"2. Helm chart for Elasticsearch"},{"line_number":125,"context_line":"3. Helm chart for Kibana"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"All three charts must include sensible configuration values to make the"},{"line_number":128,"context_line":"logging platform usable by default.  These include: proper input configurations"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_901cbbc1","line":125,"in_reply_to":"1f1a1f67_7858b92f","updated":"2017-07-27 03:56:02.000000000","message":"FYI, there is prometheus plugin at fluentd (not official one though). with that plugin, you can send metrics from fluentd to prometheus (if a specific string is showed x times within y min, send a metric to prometheus). We can probably leverage that to make log-based alarm.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"60a02d792709fdc772b14271370d18d56bb78537","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"1. Helm chart for Fluentd"},{"line_number":124,"context_line":"2. Helm chart for Elasticsearch"},{"line_number":125,"context_line":"3. Helm chart for Kibana"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"All three charts must include sensible configuration values to make the"},{"line_number":128,"context_line":"logging platform usable by default.  These include: proper input configurations"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_7858b92f","line":125,"in_reply_to":"1f1a1f67_b4f7998b","updated":"2017-07-19 10:45:58.000000000","message":"I\u0027ve noticed the same - Prometheus isn\u0027t able to provide log-based alarming infrastructure. And healthchecks stuff I was working on was supposed to deal with situations like services being unable to connect to RabbitMQ cluster or DB. That type of failure can be detected only through log analysis.\n\nI think it\u0027s worth it to at least mention log-based alarming in the spec.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":8181,"name":"Jaesuk Ahn","email":"bluejay.ahn@gmail.com","username":"blujeay.kr"},"change_message_id":"da7b2fb1e7722164617786279157112e1c6fe55d","unresolved":false,"context_lines":[{"line_number":148,"context_line":"OpenStack-Helm\u0027s alerting needs, Alertmanager can be used to interface with"},{"line_number":149,"context_line":"Prometheus and send alerts based on Prometheus rule evaluations."},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"The proposal includes the following:"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"1. Helm chart for Prometheus"},{"line_number":154,"context_line":"2. Helm chart for Alertmanager"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_5f2ecd43","line":151,"range":{"start_line":151,"start_character":26,"end_line":151,"end_character":35},"updated":"2017-07-24 15:53:07.000000000","message":"Monitoring (or logging) can probably divided into three parts; collecting, storing/querying, dashboard. Alerting can be a separate category. We can have many different tools for these purpose. Having said that, I am suggesting to change the structure of this section like the following example. In this case, we can have a generalized placeholder. (potential problem: it makes this spec too broad?)  \n\nHelm chart for metrics collection\n- prometheus chart (reference tool) \n- .. other tools can be added later .. \n\nHelm chart for metrics storage \n- influxdb chart (reference tool) \n- kafka chart \n- .. other tools can be added later ...  \n\nHelm chart for dashboard \n- grafana chart (reference tool)\n\nHelm chart for alerting (time series based) \n- prometheus alertmanager chart (reference tool) \n\nHelm chart for alerting (log based) \n- not yet","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"84bfa5c0be335d913609a1adf80589bac515c161","unresolved":false,"context_lines":[{"line_number":152,"context_line":""},{"line_number":153,"context_line":"1. Helm chart for Prometheus"},{"line_number":154,"context_line":"2. Helm chart for Alertmanager"},{"line_number":155,"context_line":"3. Helm chart for Grafana"},{"line_number":156,"context_line":"4. Helm charts for any appropriate Prometheus exporters"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"All charts must include sensible configuration values to make the monitoring"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_5f2635d0","line":155,"range":{"start_line":155,"start_character":0,"end_line":155,"end_character":25},"updated":"2017-07-18 08:27:15.000000000","message":"I see Grafana\u0027s review is abandoned. Is including it here still up to date?","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":17591,"name":"Steve Wilkerson","email":"wilkers.steve@gmail.com","username":"srwilkers"},"change_message_id":"ed221f14afbeaad2574f8e9e29b17d34afad9402","unresolved":false,"context_lines":[{"line_number":152,"context_line":""},{"line_number":153,"context_line":"1. Helm chart for Prometheus"},{"line_number":154,"context_line":"2. Helm chart for Alertmanager"},{"line_number":155,"context_line":"3. Helm chart for Grafana"},{"line_number":156,"context_line":"4. Helm charts for any appropriate Prometheus exporters"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"All charts must include sensible configuration values to make the monitoring"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_5b8520ce","line":155,"range":{"start_line":155,"start_character":0,"end_line":155,"end_character":25},"in_reply_to":"1f1a1f67_5f2635d0","updated":"2017-07-18 13:03:12.000000000","message":"yeah, it\u0027s still included going forward.  the previous review will need some updates to account for some of the new chart behaviors, but should be a decent enough base for getting started again","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"},{"author":{"_account_id":8181,"name":"Jaesuk Ahn","email":"bluejay.ahn@gmail.com","username":"blujeay.kr"},"change_message_id":"16c1a28eae9a10695b70949e21cd741bb8b29f0e","unresolved":false,"context_lines":[{"line_number":192,"context_line":"Work Items"},{"line_number":193,"context_line":"----------"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"1. Fluentd chart"},{"line_number":196,"context_line":"2. Elasticsearch chart"},{"line_number":197,"context_line":"3. Kibana chart"},{"line_number":198,"context_line":"4. Prometheus chart"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ff346bd7_d0548355","line":195,"range":{"start_line":195,"start_character":11,"end_line":195,"end_character":16},"updated":"2017-07-27 03:56:02.000000000","message":"how about adding fluentbit chart? it is very lightweight one and can have a slight different use cases from fluentd.","commit_id":"c625e154834262b3dcc905ce77c46233f3a466b9"}],"specs/osh-monitoring.rst":[{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"111a10d0fcaa97856a936af0ef8307dc9f255eb8","unresolved":false,"context_lines":[{"line_number":38,"context_line":"Alertmanager.  Prometheus, Grafana, Alertmanager, and the appropriate exporters"},{"line_number":39,"context_line":"for Prometheus provide a foundation for monitoring the cluster nodes, the"},{"line_number":40,"context_line":"cluster itself, and the containers running within the cluster to provide"},{"line_number":41,"context_line":"meaningful operational insight into an OpenStack-Helm deployment."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The proposed change includes supplying a Prometheus chart (which includes"},{"line_number":44,"context_line":"Alertmanager and kube-state-metrics), a Grafana chart for visualizing any"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3f1d235d_e0800b49","line":41,"range":{"start_line":41,"start_character":0,"end_line":41,"end_character":30},"updated":"2017-07-11 21:18:46.000000000","message":"rather than concentrating on outlining the tech solution and tools, it the spec should focus on what these meaningful operational insights that we are trying to achieve are - so we can then 1st determine if the tools are appropriate, and then 2nd deploy and configure them appropriately.","commit_id":"f9453739a1bcaac8a3004b507ef73250ec18a395"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":35,"context_line":"Some target metrics and insights into OpenStack-Helm\u0027s operations include:"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"1. Host resource utilization, such as: memory usage, CPU usage, Disk I/O, etc"},{"line_number":38,"context_line":"2. Number of replicas across deployments"},{"line_number":39,"context_line":"3. Resource consumption at the cluster, pod, and container level"},{"line_number":40,"context_line":"4. Ceph specific performance metrics, such as total pool usage or latency"},{"line_number":41,"context_line":"5. OpenStack metrics, such as tenants, networks, flavors, and floating IPs"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_ec3da4dd","line":38,"range":{"start_line":38,"start_character":3,"end_line":38,"end_character":40},"updated":"2017-07-12 01:08:50.000000000","message":"Why not have this in the same format as OpenStack/Ceph bellow - there are a lot of useful metrics we can gather from the K8s API in addition to just the replica count of a deployment :)","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"The requirements for a monitoring platform include:"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"1. Time-series database with a strong querying lanaguage interface"},{"line_number":52,"context_line":"2. Efficient storage, ideally with sharding capabilities"},{"line_number":53,"context_line":"3. Flexibility in adding new monitoring targets"},{"line_number":54,"context_line":"4. Strong visualization capabilities"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_6c299499","line":51,"range":{"start_line":51,"start_character":31,"end_line":51,"end_character":37},"updated":"2017-07-12 01:08:50.000000000","message":"superfluous","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":49,"context_line":"The requirements for a monitoring platform include:"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"1. Time-series database with a strong querying lanaguage interface"},{"line_number":52,"context_line":"2. Efficient storage, ideally with sharding capabilities"},{"line_number":53,"context_line":"3. Flexibility in adding new monitoring targets"},{"line_number":54,"context_line":"4. Strong visualization capabilities"},{"line_number":55,"context_line":"5. Ability to query data directly"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_ec16c455","line":52,"range":{"start_line":52,"start_character":3,"end_line":52,"end_character":12},"updated":"2017-07-12 01:08:50.000000000","message":"This reads like a sales pitch - surely what we are looking for is resilient storage? Regardless of how implemented.","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":51,"context_line":"1. Time-series database with a strong querying lanaguage interface"},{"line_number":52,"context_line":"2. Efficient storage, ideally with sharding capabilities"},{"line_number":53,"context_line":"3. Flexibility in adding new monitoring targets"},{"line_number":54,"context_line":"4. Strong visualization capabilities"},{"line_number":55,"context_line":"5. Ability to query data directly"},{"line_number":56,"context_line":"6. Alerting capabilities to notify operators when thresholds have been met"},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_cc0d483f","line":54,"range":{"start_line":54,"start_character":3,"end_line":54,"end_character":9},"updated":"2017-07-12 01:08:50.000000000","message":"superfluous","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":53,"context_line":"3. Flexibility in adding new monitoring targets"},{"line_number":54,"context_line":"4. Strong visualization capabilities"},{"line_number":55,"context_line":"5. Ability to query data directly"},{"line_number":56,"context_line":"6. Alerting capabilities to notify operators when thresholds have been met"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Prometheus provides the ability to monitor a target\u0027s performance via exported"},{"line_number":59,"context_line":"metrics.  This information can be queried directly through Prometheus\u0027s API or"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_8c07501d","line":56,"range":{"start_line":56,"start_character":71,"end_line":56,"end_character":74},"updated":"2017-07-12 01:08:50.000000000","message":"exceeded","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":56,"context_line":"6. Alerting capabilities to notify operators when thresholds have been met"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Prometheus provides the ability to monitor a target\u0027s performance via exported"},{"line_number":59,"context_line":"metrics.  This information can be queried directly through Prometheus\u0027s API or"},{"line_number":60,"context_line":"via a visualization tool such as Grafana, and can be used to trigger alerts via"},{"line_number":61,"context_line":"Alertmanager.  Prometheus, Grafana, Alertmanager, and the appropriate exporters"},{"line_number":62,"context_line":"for Prometheus provide a foundation for monitoring the cluster nodes, the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_b2126257","line":59,"range":{"start_line":59,"start_character":9,"end_line":59,"end_character":10},"updated":"2017-07-12 01:08:50.000000000","message":"nit: extra spaces.","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Prometheus provides the ability to monitor a target\u0027s performance via exported"},{"line_number":59,"context_line":"metrics.  This information can be queried directly through Prometheus\u0027s API or"},{"line_number":60,"context_line":"via a visualization tool such as Grafana, and can be used to trigger alerts via"},{"line_number":61,"context_line":"Alertmanager.  Prometheus, Grafana, Alertmanager, and the appropriate exporters"},{"line_number":62,"context_line":"for Prometheus provide a foundation for monitoring the cluster nodes, the"},{"line_number":63,"context_line":"cluster itself, and the containers running within the cluster to provide"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_324d3272","line":60,"range":{"start_line":60,"start_character":33,"end_line":60,"end_character":40},"updated":"2017-07-12 01:08:50.000000000","message":"are there other tools, eg Graphite/Kibana that may be of use.","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":58,"context_line":"Prometheus provides the ability to monitor a target\u0027s performance via exported"},{"line_number":59,"context_line":"metrics.  This information can be queried directly through Prometheus\u0027s API or"},{"line_number":60,"context_line":"via a visualization tool such as Grafana, and can be used to trigger alerts via"},{"line_number":61,"context_line":"Alertmanager.  Prometheus, Grafana, Alertmanager, and the appropriate exporters"},{"line_number":62,"context_line":"for Prometheus provide a foundation for monitoring the cluster nodes, the"},{"line_number":63,"context_line":"cluster itself, and the containers running within the cluster to provide"},{"line_number":64,"context_line":"insight into an OpenStack-Helm deployment."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_d224beb1","line":61,"range":{"start_line":61,"start_character":0,"end_line":61,"end_character":12},"updated":"2017-07-12 01:08:50.000000000","message":"Prometheus\u0027s Alertmanager","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"},{"author":{"_account_id":23928,"name":"Pete Birley","email":"petebirley@gmail.com","username":"portdirect"},"change_message_id":"01df35b0459561e9408a22c46be34d52f748b3b9","unresolved":false,"context_lines":[{"line_number":63,"context_line":"cluster itself, and the containers running within the cluster to provide"},{"line_number":64,"context_line":"insight into an OpenStack-Helm deployment."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"The proposed change includes supplying a Prometheus chart (which includes"},{"line_number":67,"context_line":"Alertmanager and kube-state-metrics), a Grafana chart for visualizing any"},{"line_number":68,"context_line":"exported metrics, a chart for the OpenStack exporter, and a chart for the ceph"},{"line_number":69,"context_line":"exporter.  Prometheus will scrape kube-state-metrics, the openstack-exporter,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3f1d235d_923be6c6","line":66,"range":{"start_line":66,"start_character":29,"end_line":66,"end_character":38},"updated":"2017-07-12 01:08:50.000000000","message":"developing","commit_id":"5400fe98b70aa793b304d1f8cefbbd3585094a8c"}]}
