)]}'
{".zuul.yaml":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"740a946265bf9a538276da35a904155045f35032","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"a26be624_cc74269e","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"updated":"2026-10-20 12:59:14.000000000","message":"looks like this all landed yestereday\nhttps://opendev.org/openstack/devstack-plugin-jaeger/commit/cd06ae367076d7e6baba034cfc3bffb626f17453\nso no need for a depends on.\n\nhowever the job failure is directly related to this change\n\nhttps://zuul.opendev.org/t/openstack/build/23baa664ded94339b52c1360a75f918c/log/controller/logs/screen-n-api.txt#133\n\nOct 20 09:26:37.804672 npa56044e5d08e4 devstack@n-api.service[79909]: ERROR nova.utils RuntimeError: OpenTelemetry tracing is enabled but the required packages are not installed. Install them with: pip install oslo.middleware[tracing-http] or pip install oslo.middleware[tracing-grpc]\n\nso the devstack plugin needs to install one of those.\n\nalternitivly we coudl chagne teh requirements.txt in nova however this is ment to be a optional depency so a better way to do that may be to add python extras for this for nova so you can do nova[tracing-http]","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"221366af8aa44529c5e755799b3e168f2e0c5ef7","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"f289f1af_efa794ba","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"0b1a61c7_2074cc05","updated":"2026-10-21 07:48:30.000000000","message":"As I had to respin, I have added a more detailed mention to the optional dependencies in the release note.","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"488f388188efa5ffe15cd3fdaac1683a3ae3e2dd","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"0b1a61c7_2074cc05","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"6af4d373_3689f698","updated":"2026-10-21 07:40:01.000000000","message":"Thanks for the thorough debugging and help with in your comment.\n\nI am just learning about the phases of testing. This info is invaluable.","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"63bf2ec00cbd38132bd8c074d91d63daf197a4b5","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"f638a684_c3ad14ec","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"a26be624_cc74269e","updated":"2026-10-20 13:05:03.000000000","message":"Thank you for pointing at the isse, im fixing it soon.","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e79117ec01f2639243e7c8d6259c74b7379394f6","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"6af4d373_3689f698","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"e48d12e0_f0dd21b1","updated":"2026-10-20 19:31:47.000000000","message":"ok so this time it deployed fine\n\nhttps://zuul.opendev.org/t/openstack/build/e404777712ad4bc3b1a017d3e35e3636/log/controller/logs/screen-jaeger.txt\n\nand the tests passed\n\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_e40/openstack/e404777712ad4bc3b1a017d3e35e3636/testr_results.html \n\nso the existing fucntionatliy is working\n\nhowever we do see a bunch of errors form it trying to connect to the tracinging backend\n\nhttps://zuul.opendev.org/t/openstack/build/e404777712ad4bc3b1a017d3e35e3636/log/controller/logs/screen-n-api.txt#2161-2167\n\n```\n Max retries exceeded with url: /v1/traces (Caused by NewConnectionError(\"HTTPConnection(host\u003d\u002710.176.194.66\u0027, port\u003d4318)\n```\n\nlooking at https://zuul.opendev.org/t/openstack/build/e404777712ad4bc3b1a017d3e35e3636/log/controller/logs/screen-jaeger.txt#25\n \nthe http server shoudl be listening on that port on all ipv4 interfaces\n`0.0.0.0:4318` but im not seeing any indicaton that it recived the trace attempt.\nhowever the new verifcion playbook passed\n https://zuul.opendev.org/t/openstack/build/e404777712ad4bc3b1a017d3e35e3636/log/job-output.txt#35792-35807\n \nthat means there are 2 possiblityes.\neither that playbook is incorrect and the verifcion is invilad\nor\nthe playbook/vericaiotn is valid and the devstack plugin is not starting the the backend before nova-api starts.\nthat seams more likely.\n\ndevstack has strict pahse odering and all inbuilt functionatliy alwasy runs before plugins for any given phase.\n\n\nlooking at the plugin \nhttps://review.opendev.org/c/openstack/devstack-plugin-jaeger/+/1011468/2/devstack/plugin.sh#29\n\nyou are starting jaeger in teh extra phase but it must be started in teh pos-config phase so it is started before nova is.\n\nhttps://docs.openstack.org/devstack/latest/plugins.html#plugin-sh-contract\n\nyou could argue that like rabbit it shoudl actully be considers a layer 1 service adn tehrefor should be started in the install phase but i think post-config would be early enough.\n\ncan you move\n```\n        echo_summary \"Initializing jaeger\"\n        init_jaeger\n        echo_summary \"Starting jaeger service\"\n        start_jaeger\n```\n\nform the extra phase into post-config in \n\nhttps://review.opendev.org/c/openstack/devstack-plugin-jaeger/+/1011468/2/devstack/plugin.sh#30\n\nthis might be fine to leave in extra\n```\necho_summary \"Give jaeger time to come up\"\nwait_for_data\ncheck_data_jaeger\n```\nbut you coudl move it as well to be sure its fully up before starting any of the opentack services that depend on it.\n\none other thing that might be worth doign is configring jaeger in debug mode if we can actully see logs fo the tracing coming in\n\nbut once you fix the ordering i think this is more or less good to go.","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"66acb278de11594154f9e3c63ffae563d06ae0b7","unresolved":false,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"525da32b_5db03b61","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"f289f1af_efa794ba","updated":"2026-10-22 14:04:23.000000000","message":"Done","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d4068b9b4d47d68b5e9214abfdc4d4886d2b2e65","unresolved":true,"context_lines":[{"line_number":612,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":613,"context_line":"    required-projects:"},{"line_number":614,"context_line":"        - novnc/novnc"},{"line_number":615,"context_line":"        - openstack/devstack-plugin-jaeger"},{"line_number":616,"context_line":"    roles:"},{"line_number":617,"context_line":"        # Make the verify-tracing-backend role available to the post-run"},{"line_number":618,"context_line":"        # playbook so it can assert nova-api traces landed in Jaeger."}],"source_content_type":"text/x-yaml","patch_set":10,"id":"e48d12e0_f0dd21b1","line":615,"range":{"start_line":615,"start_character":10,"end_line":615,"end_character":42},"in_reply_to":"f638a684_c3ad14ec","updated":"2026-10-20 13:24:57.000000000","message":"if you need to respin this again it woudl be good to extend the intall guide or release note to list the middlware depency but ohter then that the new devstack plugin change looks good to me.\n\nlets see what ci says.","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d1812fec45510b8872b7c7d84a3a67706d00cf5f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"8c08fbda_34d35230","updated":"2026-09-30 12:26:38.000000000","message":"-1 mainly because this change has not testing that i can see,\n\nbefore we proceed with this i woudl expect at least one of the devstack jobs to have this functionaltiy enabled","commit_id":"f2d3e91d24695c87854002879bb6884a15f0226a"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"2595450a36f3df336cfee733a01abfc6a26692d2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"31cdfb58_383e81d8","in_reply_to":"77bc1442_f94e126e","updated":"2026-10-01 07:35:44.000000000","message":"I have added a new set of tests specifically for tracing on the nova-next job.","commit_id":"f2d3e91d24695c87854002879bb6884a15f0226a"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"a59e366501206ca7b5ddaa73d663fe33221eddf4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"77bc1442_f94e126e","in_reply_to":"8c08fbda_34d35230","updated":"2026-09-30 14:03:58.000000000","message":"Thanks for the feedback Sean. Im working on it.","commit_id":"f2d3e91d24695c87854002879bb6884a15f0226a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"48adccbd2ef066ce4523d9e376f8f7d534c72837","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"22409895_ad01ae50","updated":"2026-10-01 08:23:27.000000000","message":"this is not really what i asked you to add.\n\ni asked for you to confrigure this end to end in a devstack job preferable with a real stroage backend.\n\ni was also execting you to do this by puting the logic for the devstack deploymeint into oslo-middleware so that the same impleation could be used across projects\n\nsince we ill need this for placement and any other service you want to enable oepntelemtry tracing in so even if we use the current poxy strob that shoudl be in oslo.middleware because it should also be tested in that repo.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"a167ec6b058c012a8bd10186d6262ad294fa8e34","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"d6d91a0e_bcd7452a","in_reply_to":"0f96e4a1_8971fc49","updated":"2026-10-02 11:31:25.000000000","message":"so if the feature is untested in nova then it will unsupproted by nova and same for placement cybrog watcher.\n\nany bugs with it will not be a nova bug by defiention.\n\nthe minimal testing i want to see on any repo that claims to supprot this is just to have it enabeld in a job that waht i was orgianlly asking for.\n\nany explecit assert that the backend receved trace i think are out of scope of the service project that why i described them as a nice to have but a nice to have for the feature in general not for novw.\n\nthis is sort of similar to say ironic vnc support.\n\nwe enable it in nova jobs but we initally didnt have any test for it we just traced though the debug logs to make sure it was working. later ironic added tempest test for it to have stonger validateion but we were ok to merge the ironic support once we say it workign in a job without breaking anything.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"309d06e4023b6340c072fbd94d1d587c79db10b0","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"6949fb01_80b6ce9e","in_reply_to":"18e0aa22_39a519d0","updated":"2026-10-01 17:17:39.000000000","message":"I missed Sean\u0027s intial request and was wondering why this hadn\u0027t merged yet and suddenly looked *much* larger 😀 Personally I really don\u0027t think *testing* for this feature belongs in Nova. It\u0027s an oslo.middleware feature. If need more comprehensive testing of the feature than we already have, IMO it should be done there. We don\u0027t insist on test that e.g. prove keystoneauth is able to talk to keystone.\n\nBy comparison, I\u0027ve no issues in *using* it in our jobs and testing it implicitly. Either a devstack plugin or integration into devstack proper would wfm. The latter would be less long-term maintenance, but the former would obviously be faster. Really up to you, @jlarriba@redhat.com.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"82a16f3f3580bcd277eda43ed4d8d20372aabe6b","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"29cf9f63_903cf166","in_reply_to":"22409895_ad01ae50","updated":"2026-10-01 08:28:33.000000000","message":"to be clear what i was expectign you do do was provide a devstack plugin that would deploy a backend (this simple one or preferable a real one) teak a job to enable that, have it runnign for the entirty of the job so we run tempest with this enabeld so that we catch and isseu casue by interacting with that backend, with no restart of services.\n\nthe final post hook would only assert that the traces were created in the storage backend and could be retrieved after the fact.\n\ni think that shoudl live in oslo with the middleware so it can be share between all services that use the mideelware and on that repos devstack job.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"9ef0d2ef3ff1bbc34891c54591c119aded5167e4","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"99860111_d4cfb130","in_reply_to":"29cf9f63_903cf166","updated":"2026-10-01 09:33:33.000000000","message":"Ah, I misinterpeted you, now it is clear. However, I have doubts on where to store the code for that devstack plugin. You are suggesting that it should live in oslo with the middleware, but the oslo.middleware library does not define any plugin (nor any other oslo library I searched), so it seems bold to get the plugin to live there\n\n@stephenfin@redhat.com What is your opinion about adding a devstack tracing plugin (that will operate Jaeger as the tracing backend) to oslo.middleware library?","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"03074689423e350ccab0bd2234f7715228e25c5c","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"01b297a9_2d34d2a7","in_reply_to":"46a1618f_3b598c2f","updated":"2026-10-06 09:02:06.000000000","message":"I think both of you provide reasonable arguments.\n\nIt is true that it is not under Nova to support the tracing middleware: thats on oslo. However, it is also true that oslo.middleware does not perform any end-to-end testing using a real trace storage backend during its own CI.\n\nSo, I want to have a devstack-plugin-jaeger that takes care of deploying a working tracing-enabled devstack environment for both CI and development purposes, but I also see that this Nova patch, where nothing is enabled yet, could be merged without needing that kind of end-to-end test.\n\nNo matter what, I am going to add tracing end-to-end testing to some key services, Nova being one of those, so eventually I will enable the jaeger plugin on a Nova tempest job as Sean\u0027s requesting, but it could be done in a subsequent patch, like Neutron has requested while approving this same patch: https://review.opendev.org/c/openstack/neutron/+/1007753.\n\nHowever, I could also add the jaeger testing to this patch, thats your decision to make.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"ba70c1192fb4e50a3e4437487014a02be48d8ac9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"0f96e4a1_8971fc49","in_reply_to":"6949fb01_80b6ce9e","updated":"2026-10-02 07:50:40.000000000","message":"First of all, I have removed the unwanted tests I added and left the patch as it was at the beginning.\n\nThe feature is going to be included (disabled) in every service, and I dont see it being tested in each one of them. I thought that it could be tested in just one Nova job and that would serve as a canary for if something went wrong.\n\nThe idea I was working on is to have everything packed in the devstack-plugin-jaeger (both devstack config of a backend and some verification playbooks) and get tracing enabled in one of Nova\u0027s existing tempest jobs, where the current tests run with tracing enabled (to make sure nothing breaks) and we do a quick verification at the end by searching for the traces left in Jaeger at the end of the execution.\n\nThe \"implicit testing\" idea seems to be the point where all three of us seem to align so Ill keep working on that.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"5777eb0565739419bf88aa1ef2dd1797f89f6dcd","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"ab255ff1_466a5cfa","in_reply_to":"99860111_d4cfb130","updated":"2026-10-01 11:45:22.000000000","message":"Well, without waiting for @stephenfin@redhat.com answer, I think it is better and more atomic to create a new devstack-plugin-jaeger under the telemetry umbrella to manage this plugin and use that in Nova\u0027s tempest job to collect tracing results so I am going to do that.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"dcd189f479ee52b73011463443b24a9cbe063b7a","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"18e0aa22_39a519d0","in_reply_to":"ab255ff1_466a5cfa","updated":"2026-10-01 12:30:57.000000000","message":"ya that would work too.\nand zuul alloows use to user roles or playbooks i think from requried projeect so if you added a simple assert role/playbook we could add it as a post-run playbook to any job that need it.\n\nso you could maintain the end to end testing/validation in devstack-plugin-jaeger and services can just enable it as needed","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8a70f020fb383a48657c9e18f4c8e16f8c71814a","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"46a1618f_3b598c2f","in_reply_to":"d6d91a0e_bcd7452a","updated":"2026-10-02 12:04:07.000000000","message":"\u003e so if the feature is untested in nova then it will unsupproted by nova and same for placement cybrog watcher.\n\nJuan can correct me here if I\u0027m wrong, but \"supported\" for something like this means we \"support\" the knob to enable it. We don\u0027t support the middlware itself. That\u0027s oslo\u0027s job.\n\n\u003e any bugs with it will not be a nova bug by defiention.\n\n...unless it was a bug with the knob (the `api-paste.ini` filter and `nova(-api).conf` config option, in this case). Otherwise, sure.\n\n\u003e the minimal testing i want to see on any repo that claims to supprot this is just to have it enabeld in a job that waht i was orgianlly asking for.\n\nFrom the perspective of nova, it is enabled. `otel_tracing` is added to the default middleware pipelines, so the middleware will be called. It won\u0027t do anything useful without setting the `nova.conf` option but that\u0027s okay since that\u0027s an oslo.middleware problem.\n\nhttps://github.com/openstack/oslo.middleware/blob/589d891056a832b66f2cbf7a503ff3552197d03e/oslo_middleware/tracing.py#L346-L347\n\n\u003e \n\u003e any explecit assert that the backend receved trace i think are out of scope of the service project that why i described them as a nice to have but a nice to have for the feature in general not for novw.\n\u003e \n\u003e this is sort of similar to say ironic vnc support.\n\u003e \n\u003e we enable it in nova jobs but we initally didnt have any test for it we just traced though the debug logs to make sure it was working. later ironic added tempest test for it to have stonger validateion but we were ok to merge the ironic support once we say it workign in a job without breaking anything.\n\nLooking at the [`screen-n-api.txt` logs from the most recent tempest run](https://4b21cd94afc2d673165e-79ab67b2f8928522da765be1fd16be5b.ssl.cf1.rackcdn.com/openstack/909a8686139e4ff086e42a10bb6a0c95/controller/logs/screen-n-api.txt), we can see the options are being loaded and are set to their default (disabled) values. The middleware doesn\u0027t log to show it\u0027s active though. Would the addition of a debug \"I am enabled/disabled\" log in [`TracingMiddleware.__init__`](https://github.com/openstack/oslo.middleware/blob/589d891056a832b66f2cbf7a503ff3552197d03e/oslo_middleware/tracing.py#L333) be enough so?","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"740a946265bf9a538276da35a904155045f35032","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"b6d2226e_3084a844","updated":"2026-10-20 12:59:14.000000000","message":"overall this looks good but the integration is incomplete","commit_id":"37c263fd572fd14a189afffcfee19cf359bcaf4a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"776bde38ece7af89055160e6056fddf5ee8d2561","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"96d22c62_2353a931","updated":"2026-10-21 12:57:34.000000000","message":"Drive by comment. Need to come back and review this in-depth again","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"829c9614b9ff508ae23416b264259af0d0fbe756","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"e37b7cfe_a8994b38","updated":"2026-10-22 08:22:56.000000000","message":"recheck","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"aa07184f1bcc1ea97293b5d9d398c5de14cae7f6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"f06e0561_6f43b449","updated":"2026-10-21 13:30:58.000000000","message":"recheck","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"e29bf82ee1f9e688f77a753f9633cc70c3c9e869","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"45f9ab1a_cedd6e8f","updated":"2026-10-21 11:21:05.000000000","message":"recheck an unrelated job failed","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"9ce804de7de9b94169cd4b2a0f0aea99f63aec60","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"f3055aec_e11914d3","updated":"2026-10-23 08:51:30.000000000","message":"recheck depending patch merged","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"66acb278de11594154f9e3c63ffae563d06ae0b7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"45b6ea19_8971d4df","updated":"2026-10-22 14:04:23.000000000","message":"the errors are gone, tempest reuslts are green and the verficaion teas seemed to run properly so looks good to me thansk for adding that.","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"}],"releasenotes/notes/otel-tracing-middleware-f8074d3bfe589ee7.yaml":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"776bde38ece7af89055160e6056fddf5ee8d2561","unresolved":true,"context_lines":[{"line_number":6,"context_line":"    pipeline and is disabled by default, acting as a zero-overhead passthrough"},{"line_number":7,"context_line":"    when off. To enable distributed tracing, set ``enabled \u003d True`` under the"},{"line_number":8,"context_line":"    ``[oslo_middleware_tracing]`` group in ``nova.conf`` and install the"},{"line_number":9,"context_line":"    optional OpenTelemetry exporter packages, which ship as extras of"},{"line_number":10,"context_line":"    ``oslo.middleware``: ``oslo.middleware[tracing-http]`` for the OTLP HTTP"},{"line_number":11,"context_line":"    exporter or ``oslo.middleware[tracing-grpc]`` for the OTLP gRPC exporter."},{"line_number":12,"context_line":"    Enabling the middleware without these packages installed will cause the"},{"line_number":13,"context_line":"    service to fail at startup."}],"source_content_type":"text/x-yaml","patch_set":12,"id":"035b7203_e99c8be7","line":13,"range":{"start_line":9,"start_character":69,"end_line":13,"end_character":31},"updated":"2026-10-21 12:57:34.000000000","message":"Do we want to provide our own extras for this?","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":32968,"name":"Juan Larriba","email":"jlarriba@redhat.com","username":"jlarriba"},"change_message_id":"08694980c1f1c3f5561c22dc4e8158dc8716a675","unresolved":true,"context_lines":[{"line_number":6,"context_line":"    pipeline and is disabled by default, acting as a zero-overhead passthrough"},{"line_number":7,"context_line":"    when off. To enable distributed tracing, set ``enabled \u003d True`` under the"},{"line_number":8,"context_line":"    ``[oslo_middleware_tracing]`` group in ``nova.conf`` and install the"},{"line_number":9,"context_line":"    optional OpenTelemetry exporter packages, which ship as extras of"},{"line_number":10,"context_line":"    ``oslo.middleware``: ``oslo.middleware[tracing-http]`` for the OTLP HTTP"},{"line_number":11,"context_line":"    exporter or ``oslo.middleware[tracing-grpc]`` for the OTLP gRPC exporter."},{"line_number":12,"context_line":"    Enabling the middleware without these packages installed will cause the"},{"line_number":13,"context_line":"    service to fail at startup."}],"source_content_type":"text/x-yaml","patch_set":12,"id":"87f26a8c_1f486641","line":13,"range":{"start_line":9,"start_character":69,"end_line":13,"end_character":31},"in_reply_to":"035b7203_e99c8be7","updated":"2026-10-22 08:24:27.000000000","message":"Is this question for me?\n\nI dont really know what to answer. Seems quite reasonable to me to keep the most amount of things tracing-related in the plugin so the changes to Nova are the smallest possible.","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"66acb278de11594154f9e3c63ffae563d06ae0b7","unresolved":true,"context_lines":[{"line_number":6,"context_line":"    pipeline and is disabled by default, acting as a zero-overhead passthrough"},{"line_number":7,"context_line":"    when off. To enable distributed tracing, set ``enabled \u003d True`` under the"},{"line_number":8,"context_line":"    ``[oslo_middleware_tracing]`` group in ``nova.conf`` and install the"},{"line_number":9,"context_line":"    optional OpenTelemetry exporter packages, which ship as extras of"},{"line_number":10,"context_line":"    ``oslo.middleware``: ``oslo.middleware[tracing-http]`` for the OTLP HTTP"},{"line_number":11,"context_line":"    exporter or ``oslo.middleware[tracing-grpc]`` for the OTLP gRPC exporter."},{"line_number":12,"context_line":"    Enabling the middleware without these packages installed will cause the"},{"line_number":13,"context_line":"    service to fail at startup."}],"source_content_type":"text/x-yaml","patch_set":12,"id":"a6e76905_f60a454d","line":13,"range":{"start_line":9,"start_character":69,"end_line":13,"end_character":31},"in_reply_to":"87f26a8c_1f486641","updated":"2026-10-22 14:04:23.000000000","message":"it proably is a good idea to do this but im ok with leaving that to a followup\n\nbasiclly nova[tracing-http] would just install oslo.middleware[tracing-http] for you \n\nwe proably woudl not end up usign that in devstack but we could supprot it via 2 new intrties in \n\nhttps://github.com/openstack/nova/blob/master/pyproject.toml#L31-L40","commit_id":"eec75af5ce9b202c04ebc96aa4cb60aceb50b7c1"}],"roles/run-nova-tracing-tests/files/verify_tracing.py":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"48adccbd2ef066ce4523d9e376f8f7d534c72837","unresolved":true,"context_lines":[{"line_number":45,"context_line":"            for item in values}"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"def tracing_config(original, endpoint):"},{"line_number":49,"context_line":"    \"\"\"Change tracing options without rewriting unrelated/repeated options.\"\"\""},{"line_number":50,"context_line":"    options \u003d {\u0027enabled\u0027, \u0027otlp_protocol\u0027, \u0027otlp_endpoint\u0027, \u0027sampling_rate\u0027,"},{"line_number":51,"context_line":"               \u0027service_name\u0027}"},{"line_number":52,"context_line":"    section \u003d None"},{"line_number":53,"context_line":"    skipping \u003d False"},{"line_number":54,"context_line":"    result \u003d []"},{"line_number":55,"context_line":"    for line in original.splitlines(keepends\u003dTrue):"},{"line_number":56,"context_line":"        header \u003d re.match(r\u0027^\\s*\\[([^]]+)\\]\u0027, line)"},{"line_number":57,"context_line":"        if header:"},{"line_number":58,"context_line":"            section \u003d header.group(1)"},{"line_number":59,"context_line":"            skipping \u003d False"},{"line_number":60,"context_line":"        option \u003d re.match(r\u0027^\\s*([^#;\\s\u003d]+)\\s*\u003d\u0027, line)"},{"line_number":61,"context_line":"        if section \u003d\u003d GROUP and option:"},{"line_number":62,"context_line":"            skipping \u003d option.group(1).lower() in options"},{"line_number":63,"context_line":"            if skipping:"},{"line_number":64,"context_line":"                continue"},{"line_number":65,"context_line":"        elif skipping and line[:1].isspace() and line.strip():"},{"line_number":66,"context_line":"            continue"},{"line_number":67,"context_line":"        else:"},{"line_number":68,"context_line":"            skipping \u003d False"},{"line_number":69,"context_line":"        result.append(line)"},{"line_number":70,"context_line":"    result.append(\u0027\\n[%s]\\nenabled \u003d true\\notlp_protocol \u003d http/protobuf\\n\u0027"},{"line_number":71,"context_line":"                  \u0027otlp_endpoint \u003d %s\\nsampling_rate \u003d 1.0\\n\u0027 %"},{"line_number":72,"context_line":"                  (GROUP, endpoint))"},{"line_number":73,"context_line":"    return \u0027\u0027.join(result)"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"class Collector(http.server.ThreadingHTTPServer):"}],"source_content_type":"text/x-python","patch_set":8,"id":"f6e6f3c5_99c7f84a","line":73,"range":{"start_line":48,"start_character":1,"end_line":73,"end_character":26},"updated":"2026-10-01 08:23:27.000000000","message":"why do it like this?\n\nyou could just use crudini or even git config set --file ...\n\nor if you want to do this in python then you could trivially jsut use configparser \n\nto add the section to the config.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"48adccbd2ef066ce4523d9e376f8f7d534c72837","unresolved":true,"context_lines":[{"line_number":72,"context_line":"                  (GROUP, endpoint))"},{"line_number":73,"context_line":"    return \u0027\u0027.join(result)"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"class Collector(http.server.ThreadingHTTPServer):"},{"line_number":77,"context_line":"    daemon_threads \u003d True"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"    def __init__(self):"},{"line_number":80,"context_line":"        super().__init__((\u0027127.0.0.1\u0027, 0), ExportHandler)"},{"line_number":81,"context_line":"        self.condition \u003d threading.Condition()"},{"line_number":82,"context_line":"        self.spans \u003d []"},{"line_number":83,"context_line":"        self.errors \u003d []"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"    def matching_span(self, trace_id, timeout):"},{"line_number":86,"context_line":"        deadline \u003d time.monotonic() + timeout"},{"line_number":87,"context_line":"        with self.condition:"},{"line_number":88,"context_line":"            while True:"},{"line_number":89,"context_line":"                require(not self.errors,"},{"line_number":90,"context_line":"                        \u0027OTLP receiver rejected an export: %s\u0027 % self.errors)"},{"line_number":91,"context_line":"                matches \u003d [(resource, span) for resource, span in self.spans"},{"line_number":92,"context_line":"                           if span.trace_id.hex() \u003d\u003d trace_id]"},{"line_number":93,"context_line":"                if matches:"},{"line_number":94,"context_line":"                    require(len(matches) \u003d\u003d 1,"},{"line_number":95,"context_line":"                            \u0027Expected one server span for trace %s\u0027 % trace_id)"},{"line_number":96,"context_line":"                    return matches[0]"},{"line_number":97,"context_line":"                remaining \u003d deadline - time.monotonic()"},{"line_number":98,"context_line":"                require(remaining \u003e 0,"},{"line_number":99,"context_line":"                        \u0027No exported span for trace %s\u0027 % trace_id)"},{"line_number":100,"context_line":"                self.condition.wait(remaining)"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"class ExportHandler(http.server.BaseHTTPRequestHandler):"},{"line_number":104,"context_line":"    def do_POST(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"2618f5c2_34b23905","line":101,"range":{"start_line":75,"start_character":0,"end_line":101,"end_character":0},"updated":"2026-10-01 08:23:27.000000000","message":"why are you implenting a fake http server for this instead of using a real backend?\n\ni was expecting you to modify devstack to deploy whatever real service the logs are indended to be set too.","commit_id":"7dcb3ed74fb35f5788be43cb8954e2aa89de7ef5"}]}