)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":9976,"name":"Ronelle Landy","email":"rlandy@redhat.com","username":"rlandy"},"change_message_id":"bf3d9fe3d419c09ed9aecaa355cd99a672e68e01","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"16259fc5_a63ebd1e","updated":"2025-02-27 19:05:11.000000000","message":"agreed on approach","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"7b2ce3271479a7ba89b6f75b662e6534c28a933e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"83439110_5772f5b2","updated":"2025-02-25 07:14:42.000000000","message":"failing jobs are hitting following error https://0b69c48f785a4a97cd1c-76bf72ffc642f12cb8e7d8393148d522.ssl.cf2.rackcdn.com/942481/5/check/watcher-tempest-functional/816a7ac/job-output.txt\n```\n++ /opt/stack/watcher/devstack/lib/watcher:init_watcher:257 :   /opt/stack/data/venv/bin/watcher-db-manage --config-file /etc/watcher/watcher.conf upgrade\n2025-02-24 20:13:47.365274 | controller | Traceback (most recent call last):\n2025-02-24 20:13:47.365319 | controller |   File \"/opt/stack/data/venv/bin/watcher-db-manage\", line 6, in \u003cmodule\u003e\n2025-02-24 20:13:47.365331 | controller |     from watcher.cmd.dbmanage import main\n2025-02-24 20:13:47.365341 | controller |   File \"/opt/stack/watcher/watcher/cmd/dbmanage.py\", line 26, in \u003cmodule\u003e\n2025-02-24 20:13:47.365351 | controller |     from watcher.common import service\n2025-02-24 20:13:47.365364 | controller |   File \"/opt/stack/watcher/watcher/common/service.py\", line 67, in \u003cmodule\u003e\n2025-02-24 20:13:47.365515 | controller |     Singleton \u003d service.Singleton\n2025-02-24 20:13:47.365617 | controller |                 ^^^^^^^^^^^^^^^^^\n2025-02-24 20:13:47.365696 | controller | AttributeError: module \u0027oslo_service.service\u0027 has no attribute \u0027Singleton\u0027\n```\nhttps://github.com/openstack/oslo.service/commit/861e8d6677df63660268376fcf4a78b2838d5ad9#diff-086b3143a7ffd2a89856a6a38ae7320c639830d3801116626d3d1fc57a3e886eL116 removes Singleton class and https://review.opendev.org/c/openstack/requirements/+/942447 bumps update constraint for oslo.service to new release 4.1.0 broke the CI.","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"fce54d8ea92038b24c2ef1cf363e23e9db1f9538","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"db80c2ff_2d0843b8","updated":"2025-02-27 15:24:40.000000000","message":"recheck","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"202a98fde4a17ecf708b1821cf2be457f8e0b639","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"cbae7b1b_5c8a344b","updated":"2025-02-25 12:10:20.000000000","message":"recheck after https://review.opendev.org/c/openstack/oslo.service/+/942666 merged","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"ebe83040197fc29270c529d3fbd96f236723961f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"23daac67_944fa7a5","updated":"2025-02-27 09:34:23.000000000","message":"recheck https://review.opendev.org/c/openstack/requirements/+/942830 merged","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":16312,"name":"Alfredo Moralejo","email":"amoralej@redhat.com","username":"amoralej"},"change_message_id":"0e0669de571b66ccdfd82dca25088cffefdbaf08","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"11f70a79_925101de","updated":"2025-02-28 11:10:25.000000000","message":"tested in https://github.com/openstack-k8s-operators/watcher-operator/pull/97","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"763db366b1c44d78fc19f648e40df3f5524a05a6","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"2f11c789_e1d4d576","in_reply_to":"11f70a79_925101de","updated":"2025-02-28 12:53:57.000000000","message":"looking at that more closely \n\ni see that prometious is correctly enabeld in the plugin for telemetry\n\nhttps://softwarefactory-project.io/zuul/t/rdoproject.org/build/8942bd50e18b443a9c6a2cbbdf42d87c/log/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/etc/tempest.conf#133\n\n\nbut not for watcher\n\nhttps://softwarefactory-project.io/zuul/t/rdoproject.org/build/8942bd50e18b443a9c6a2cbbdf42d87c/log/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/etc/tempest.conf#140\n\n\nthe regex is executing tests that should require metrics and they pass.\n\nhttps://logserver.rdoproject.org/97/97/a8c2e7a71756a1dda1995202bad30c362c72a008/github-check/watcher-operator-validation/8942bd5/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/stestr_results.html\n\nbut with no datasouce enabled they should fail\n\n\nif we look at the tempest logs\n\nhttps://softwarefactory-project.io/zuul/t/rdoproject.org/build/8942bd50e18b443a9c6a2cbbdf42d87c/log/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/tempest.log#594\n\n\n2025-02-28 09:49:34.389 109 DEBUG tempest [-] profiler.key                   \u003d None log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.389 109 DEBUG tempest [-] optimize.datasource            \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.389 109 DEBUG tempest [-] optimize.openstack_type        \u003d devstack log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.podified_kubeconfig_path \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.podified_namespace    \u003d openstack log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_fqdn_label \u003d fqdn log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_host       \u003d 127.0.0.1 log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_port       \u003d 9090 log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_promtool   \u003d promtool log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_ssl_cert_dir \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.prometheus_ssl_enabled \u003d False log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.proxy_host_address    \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.390 109 DEBUG tempest [-] optimize.proxy_host_pkey       \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.391 109 DEBUG tempest [-] optimize.proxy_host_pkey_type  \u003d rsa log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n2025-02-28 09:49:34.391 109 DEBUG tempest [-] optimize.proxy_host_user       \u003d  log_opt_values /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/oslo_config/cfg.py:2609\n\n\ni am not seeign the config  either that was specified here \n\nhttps://github.com/openstack-k8s-operators/watcher-operator/pull/97/files#diff-108978819c05ae183d88ec87959c2341a94cfc3f9465e3aeee82d554217b4f58R205\n\nwe do see in the logs too \"Configuring PromtoolClient for devstack  deployment\"\nwhich more or less confrims that job did nto test what we wanted it to test\n\nthat is printed here.\nhttps://review.opendev.org/c/openstack/watcher-tempest-plugin/+/942481/5/watcher_tempest_plugin/services/metric/prometheus_client.py#37\n\n\nso https://github.com/openstack-k8s-operators/watcher-operator/pull/97/files#diff-108978819c05ae183d88ec87959c2341a94cfc3f9465e3aeee82d554217b4f58R205 did not properly validate this","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7ebf7e0c8f1771239f5529ff664ee4fd6467790a","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"4c234685_a3b0e313","in_reply_to":"2f11c789_e1d4d576","updated":"2025-02-28 13:07:33.000000000","message":"ok the changes were made to the master version instead.\n\nso doing this again\n\nthis teime we do see the plugin config options set in the tempest config\n\nhttps://softwarefactory-project.io/zuul/t/rdoproject.org/build/b2979b17bb0044d4a9743f21f625aa86/log/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/etc/tempest.conf#139-148\n\nand the setting corectly proagated into the tempest execution\nhttps://softwarefactory-project.io/zuul/t/rdoproject.org/build/b2979b17bb0044d4a9743f21f625aa86/log/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/tempest.log#594\n\nwe can see it was able to correctly ssh into the contoler and lookup the prometius pod name\n\n2025-02-28 09:57:36.452 108 DEBUG watcher_tempest_plugin.services.metric.prometheus_client [-] Prometheus service found. Running on pod prometheus-metric-storage-0 get_prometheus_pod \n\nand we can see it usign the correct promtool client\n\n2025-02-28 09:59:38.150 108 DEBUG watcher_tempest_plugin.services.metric.prometheus_client [-] Configuring PromtoolClient for podified  deployment. __init__ /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/watcher_tempest_plugin/services/metric/prometheus_client.py:37\n\n\nwe do have a failure of one test\n\nhttps://logserver.rdoproject.org/97/97/a8c2e7a71756a1dda1995202bad30c362c72a008/github-check/watcher-operator-validation-master/b2979b1/controller/ci-framework-data/tests/test_operator/tempest-tests-tempest/stestr_results.html\n\nhowever this did not run with the \"wait for model to be updated\" change so that looks unrelated\n\nbased on that im happy to appove.","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"482c6fbafc13a8ccafc5f3a38f1179dd8114cf3c","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"d529ccd4_be4fa250","in_reply_to":"4c234685_a3b0e313","updated":"2025-02-28 13:10:10.000000000","message":"we dont recored the command we executed in the logs\n\nbut we do log the output of the command execution \n\nhttps://review.opendev.org/c/openstack/watcher-tempest-plugin/+/942481/5/watcher_tempest_plugin/services/metric/prometheus_client.py#130\n\n2025-02-28 10:00:46.648 108 DEBUG watcher_tempest_plugin.services.metric.prometheus_client [-] Promtool add_measures output: Parsing standard input\n\n  SUCCESS: metrics pushed to remote write.\n\n add_measures /var/lib/tempest/external-plugins/.venv/lib64/python3.9/site-packages/watcher_tempest_plugin/services/metric/prometheus_client.py:130","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"994e774eac8f0038220885a74652bd7578c4c0d7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"d7d6bab1_27641095","in_reply_to":"83439110_5772f5b2","updated":"2025-02-25 07:29:15.000000000","message":"Reported it here: https://bugs.launchpad.net/watcher/+bug/2099968","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"f68bdcc9f47a84c52220e44470970e529e992ea9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"d6f5fcde_014bf75f","in_reply_to":"d7d6bab1_27641095","updated":"2025-02-25 07:44:16.000000000","message":"Proposed https://review.opendev.org/c/openstack/watcher/+/942674 to address the issue or Can we pin the oslo.service version for now?","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"}],"watcher_tempest_plugin/services/metric/prometheus_client.py":[{"author":{"_account_id":16312,"name":"Alfredo Moralejo","email":"amoralej@redhat.com","username":"amoralej"},"change_message_id":"5fee641556d1a7e9d4834d9c4802f1992f96b17c","unresolved":true,"context_lines":[{"line_number":36,"context_line":"        self.is_podified \u003d (\"podified\" \u003d\u003d openstack_type)"},{"line_number":37,"context_line":"        LOG.debug(f\"Configuring PromtoolClient for {openstack_type} \""},{"line_number":38,"context_line":"                  \" deployment.\")"},{"line_number":39,"context_line":"        self.podified_ns \u003d podified_ns"},{"line_number":40,"context_line":"        self.oc_cmd \u003d [\u0027oc\u0027]"},{"line_number":41,"context_line":"        if podified_kubeconfig:"},{"line_number":42,"context_line":"            self.oc_cmd.insert(0, f\"KUBECONFIG\u003d{podified_kubeconfig}\")"},{"line_number":43,"context_line":"        if podified_ns:"},{"line_number":44,"context_line":"            self.oc_cmd +\u003d [\u0027-n\u0027, self.podified_ns]"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"        if proxy_host_address and proxy_host_user:"},{"line_number":47,"context_line":"            self.client \u003d base.SshCmdClient("}],"source_content_type":"text/x-python","patch_set":5,"id":"85e13fc3_1a184800","line":44,"range":{"start_line":39,"start_character":1,"end_line":44,"end_character":51},"updated":"2025-02-27 16:29:24.000000000","message":"nit. I guess this is is only meaninful if is_podified, for clarity, it may go to conditional in line 56","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":30002,"name":"Douglas Viroel","email":"viroel@gmail.com","username":"dviroel"},"change_message_id":"629985efb5042e247afd127212276a3de4c13d17","unresolved":true,"context_lines":[{"line_number":36,"context_line":"        self.is_podified \u003d (\"podified\" \u003d\u003d openstack_type)"},{"line_number":37,"context_line":"        LOG.debug(f\"Configuring PromtoolClient for {openstack_type} \""},{"line_number":38,"context_line":"                  \" deployment.\")"},{"line_number":39,"context_line":"        self.podified_ns \u003d podified_ns"},{"line_number":40,"context_line":"        self.oc_cmd \u003d [\u0027oc\u0027]"},{"line_number":41,"context_line":"        if podified_kubeconfig:"},{"line_number":42,"context_line":"            self.oc_cmd.insert(0, f\"KUBECONFIG\u003d{podified_kubeconfig}\")"},{"line_number":43,"context_line":"        if podified_ns:"},{"line_number":44,"context_line":"            self.oc_cmd +\u003d [\u0027-n\u0027, self.podified_ns]"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"        if proxy_host_address and proxy_host_user:"},{"line_number":47,"context_line":"            self.client \u003d base.SshCmdClient("}],"source_content_type":"text/x-python","patch_set":5,"id":"ba0eaae8_be39fd85","line":44,"range":{"start_line":39,"start_character":1,"end_line":44,"end_character":51},"in_reply_to":"85e13fc3_1a184800","updated":"2025-02-27 16:47:41.000000000","message":"yes, makes sense, I can propose this in a follow up PS or new change. Leaving as unresolved for now.","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":16312,"name":"Alfredo Moralejo","email":"amoralej@redhat.com","username":"amoralej"},"change_message_id":"5fee641556d1a7e9d4834d9c4802f1992f96b17c","unresolved":true,"context_lines":[{"line_number":80,"context_line":"    def _build_host_fqdn_maps(self):"},{"line_number":81,"context_line":"        # NOTE(dviroel): Promtool does not support \u0027targets\u0027"},{"line_number":82,"context_line":"        # endpoint."},{"line_number":83,"context_line":"        cmd \u003d [\u0027curl\u0027, \u0027-k\u0027, \u0027-s\u0027, self.prometheus_targets_url]"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"        out \u003d self.client.exec_cmd(cmd)"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"        targets \u003d json.loads(out)[\u0027data\u0027][\u0027activeTargets\u0027]"},{"line_number":88,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"1980f518_17e9f5d7","line":85,"range":{"start_line":83,"start_character":0,"end_line":85,"end_character":39},"updated":"2025-02-27 16:29:24.000000000","message":"I understand you do this with curl instead of a simple http call with a python http client because we may not reach prometheus from the environment running tempest? otherwise, I\u0027d do in a more pythonic way.","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"},{"author":{"_account_id":30002,"name":"Douglas Viroel","email":"viroel@gmail.com","username":"dviroel"},"change_message_id":"629985efb5042e247afd127212276a3de4c13d17","unresolved":true,"context_lines":[{"line_number":80,"context_line":"    def _build_host_fqdn_maps(self):"},{"line_number":81,"context_line":"        # NOTE(dviroel): Promtool does not support \u0027targets\u0027"},{"line_number":82,"context_line":"        # endpoint."},{"line_number":83,"context_line":"        cmd \u003d [\u0027curl\u0027, \u0027-k\u0027, \u0027-s\u0027, self.prometheus_targets_url]"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"        out \u003d self.client.exec_cmd(cmd)"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"        targets \u003d json.loads(out)[\u0027data\u0027][\u0027activeTargets\u0027]"},{"line_number":88,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"83f3042a_1c7f2aef","line":85,"range":{"start_line":83,"start_character":0,"end_line":85,"end_character":39},"in_reply_to":"1980f518_17e9f5d7","updated":"2025-02-27 16:47:41.000000000","message":"Correct, in the podified deployment this is going to run inside prometheus container, through the proxy host, which makes more difficult.","commit_id":"07a8a50f67e877e486441b512a1226bcb62deec7"}]}