)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":34975,"name":"Jaromír Wysoglad","email":"jwysogla@redhat.com","username":"jwysogla"},"change_message_id":"7cd5a2b30d3b5c7a9664a32d9c45589d901ab72f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"103d2243_f4294053","updated":"2024-10-24 14:02:32.000000000","message":"I tried to find a solution to this myself and I couldn\u0027t figure out how to have:\n - correct value\n - \"counter\" as a metric type\n - and not having \"_total\" added to the metric names.\n\nUsing gauge for metric type seems like the best solution since Prometheus doesn\u0027t care about types anyway. LGTM.","commit_id":"073044fc28e682df40e7249e09a115f97df2ed0e"},{"author":{"_account_id":5202,"name":"Erno Kuvaja","email":"jokke@usr.fi","username":"jokke"},"change_message_id":"361d6c0bce93c8b902f1e67fbd64e1c118e3bdeb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"c29a9449_f3c561fe","updated":"2024-11-19 11:40:48.000000000","message":"Could you please modify the tests to actually verify the \u0027cumulative\u0027 produces correct result this time. The test data is in place it\u0027s just never verified to produce the desired outcome https://github.com/openstack/ceilometer/blob/master/ceilometer/tests/unit/test_prom_exporter.py#L35","commit_id":"7b3ff3b504dc113f967f624fdaa1c50a46943ec2"},{"author":{"_account_id":5202,"name":"Erno Kuvaja","email":"jokke@usr.fi","username":"jokke"},"change_message_id":"edc31bed1a175e5d56a03311f6257d8b1d39b2ac","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"41efa868_1cb34bfa","updated":"2024-11-21 15:17:09.000000000","message":"LGTM, thanks Juan","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"e14bde3a19ac724f545b0c152ae54fcb653a1ce6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"2a2ab659_5b0014a4","updated":"2024-11-22 18:31:06.000000000","message":"LGTM. We can continue the discussion somewhere (maybe in irc ?) but that\u0027s independent from this change so I\u0027m merging this now.\n\nnote for my self: The prometheus exporter code was added in this cycle so we don\u0027t have to care about backport incompatibility.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"}],"ceilometer/polling/prom_exporter.py":[{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"883a71c212768ad19e6dfefbbe6f42d4a2a84d53","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"004a3295_bc001fed","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"updated":"2024-11-20 14:56:17.000000000","message":"side note: the current implementation does not unregister metrics, which means that if you migrate a server between two hosts then metric for that server may be kept in the old host, which may cause some confusions.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"d90b0e0795572b75b7f80a21558deaa2e852263c","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"3ca4774a_2321dfda","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"004a3295_bc001fed","updated":"2024-11-20 16:34:30.000000000","message":"That behaviour is good and wanted, as the metrics correctly state that until a certain time today that server was on a host and since it is in a different one. It gives very useful information to the user.\n\nAnd at a certain point, the older metrics will disappear.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"e14bde3a19ac724f545b0c152ae54fcb653a1ce6","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"76a416dd_95d298ed","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"113aa192_15cf2b64","updated":"2024-11-22 18:31:06.000000000","message":"I have the following scenario in my mind.\n 1. VM A is first launched at host A\n 2. VM A\u0027s metrics are registered in host A and are exposed in target endpoint of host A\n 3. VM A is migated to host B\n 4. VM A\u0027s metrics are now registered in host B and are exposed in target endpoint of host B\n\nThe metrics registered in 2 are not removed at step 3, because the metrics are never un-registered from the registry so the old data may exist in target endpoint of host A after 3. Prometheus may pull metrics from endpoints of both two nodes, but it does not know if the metrics in host A are stale and it still continues adding the old data into its TSDB in every polling. The problem here is that prometheus may generate new metric record, with the new timestamp, according to the time when that polling is executed. So \"new metrics\" may be continuously created with old metrics data.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"8b7dddefad4f9ff5279b93551d992b61b9315937","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"4cc0c488_8400c6fa","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"3ab7be20_8025b819","updated":"2024-11-25 11:02:06.000000000","message":"You can find me in IRC so we could talk more in detail about this. However, I want to highlight that what you described is not how Prometheus works. You are assuming that Prometheus assigns a timestamp to the retrieved metric when it scrapes it, but it does not. Every metric is a tuple of value and timestamp.\n\nIn the case you describe, the metrics will be actually exposed in both endpoint A and B, but the A metrics will be \"old\", as their timestamp will never change, while node A is not VM generating metrics for that specific VM.\n\nPrometheus knows this and it will not store anything new coming from host A (becaue there are no new metrics there), all new metrics will be coming from host B. With this behaviour, the user will be able to see in his dashboard a metric from the VM from host A that suddendly stops, just in time for a new metric from the VM from host B appearing on his dashboards, creating a \"single metric\" accros a metric with different labels.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"5911a57f467d313062d00263921dc46304446ba8","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"6be08453_72d9bd0d","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"3ca4774a_2321dfda","updated":"2024-11-20 16:39:56.000000000","message":"An example problem you may face is that when you try to set autoscaling based on cpu metrics you have to make sure that only the metrics from the host where the instance is assigned is used. Otherwise the evaluation result may be messed up by the \"stale\" metrics obtained from the original host.\n\n\u003e And at a certain point, the older metrics will disappear.\nNow the metric is not deleted unless you restart the agent, right ?","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"419346382b2916536101e4829b605a606b3426ea","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"113aa192_15cf2b64","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"6be08453_72d9bd0d","updated":"2024-11-20 16:46:59.000000000","message":"I am afraid I am not understanding. There is no possibility of two metrics being generated for the same VM from different hosts at the same time: they will follow a time series. \n\nIf autoscaling did not triggered with the \"stale\" metrics while the VM was in the old node, it will not pay attention to them after \"new\" metrics start being generated, as the \"stale\" metrics will inevitably be older than the \"new\" metrics.\n\nAm I not understanding a specific use case here?","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"b3e5fc320fb7d3e12c60029d1c2d24eaa3f5615e","unresolved":false,"context_lines":[{"line_number":30,"context_line":"        name \u003d \"ceilometer_\" + sample[\u0027counter_name\u0027].replace(\u0027.\u0027, \u0027_\u0027)"},{"line_number":31,"context_line":"        labels \u003d _gen_labels(sample)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"        metric \u003d CEILOMETER_REGISTRY._names_to_collectors.get(name, None)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"        if metric is None:"},{"line_number":36,"context_line":"            metric \u003d prom.Gauge(name\u003dname, documentation\u003d\"\","}],"source_content_type":"text/x-python","patch_set":3,"id":"3ab7be20_8025b819","line":33,"range":{"start_line":33,"start_character":17,"end_line":33,"end_character":36},"in_reply_to":"76a416dd_95d298ed","updated":"2024-11-22 18:36:31.000000000","message":"\u003e So \"new metrics\" may be continuously created with old metrics data.\n\nI mean that prometheus may pull both stale data from host A and new data from host B, and created TSDB records for both, with updated timestamp. So we may end up having the records of the same vm (VM A), from different hosts(host A and host B), continuously created in every polling.","commit_id":"f8af14c235b898657746d3ef278d0c32c1408a25"}]}