)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b04bcc4090c3fd5823aa78247ce7296dea596acc","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix default values for variables in run-tempest role"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"With the new version of tox the parsing of postargs value has changed."},{"line_number":10,"context_line":"Previously an empty string was discarded by tox if it appeared in the"},{"line_number":11,"context_line":"postargs value in contrast with the latest version of tox which"},{"line_number":12,"context_line":"leaves the empty string intact."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The behavior of the latest tox version makes some jobs fail as some"},{"line_number":15,"context_line":"variables in the run-tempest role have an empty string as a default"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"62795575_89540a48","line":12,"range":{"start_line":9,"start_character":0,"end_line":12,"end_character":31},"updated":"2023-02-02 19:12:44.000000000","message":"do we have link of the tox behavior change which will be helpful to know about it.","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"9fa2bdd5fdc8ccdfc3ccd3e13bef27104bfea641","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix default values for variables in run-tempest role"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"With the new version of tox the parsing of postargs value has changed."},{"line_number":10,"context_line":"Previously an empty string was discarded by tox if it appeared in the"},{"line_number":11,"context_line":"postargs value in contrast with the latest version of tox which"},{"line_number":12,"context_line":"leaves the empty string intact."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The behavior of the latest tox version makes some jobs fail as some"},{"line_number":15,"context_line":"variables in the run-tempest role have an empty string as a default"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"d97ebbe2_a2a006d5","line":12,"range":{"start_line":9,"start_character":0,"end_line":12,"end_character":31},"in_reply_to":"62795575_89540a48","updated":"2023-02-03 09:30:55.000000000","message":"I went through the release notes of Tox but I couldn\u0027t find any mention of this specific change. The change was made somewhere between 3.28.0 (the last 3.x.x version) and 4.0.0. There were a lot of changes between those two versions so I guess it got lost between all the patches.","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"9fa2bdd5fdc8ccdfc3ccd3e13bef27104bfea641","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix default values for variables in run-tempest role"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"With the new version of tox the parsing of postargs value has changed."},{"line_number":10,"context_line":"Previously an empty string was discarded by tox if it appeared in the"},{"line_number":11,"context_line":"postargs value in contrast with the latest version of tox which"},{"line_number":12,"context_line":"leaves the empty string intact."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The behavior of the latest tox version makes some jobs fail as some"},{"line_number":15,"context_line":"variables in the run-tempest role have an empty string as a default"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"b097520c_35344700","line":12,"range":{"start_line":9,"start_character":0,"end_line":12,"end_character":31},"in_reply_to":"62795575_89540a48","updated":"2023-02-03 09:30:55.000000000","message":"It must have happened somewhere between 3.28.0 and 4.0.0. I went through the release notes but there is was no mention of this. I guess","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"51bea16980a43e3e70990770c2ad428e53c218bb","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix default values for variables in run-tempest role"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"With the new version of tox the parsing of postargs value has changed."},{"line_number":10,"context_line":"Previously an empty string was discarded by tox if it appeared in the"},{"line_number":11,"context_line":"postargs value in contrast with the latest version of tox which"},{"line_number":12,"context_line":"leaves the empty string intact."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The behavior of the latest tox version makes some jobs fail as some"},{"line_number":15,"context_line":"variables in the run-tempest role have an empty string as a default"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"f8fa62af_9d199e52","line":12,"range":{"start_line":9,"start_character":0,"end_line":12,"end_character":31},"in_reply_to":"d97ebbe2_a2a006d5","updated":"2023-02-03 23:36:13.000000000","message":"yeah, I also did not find anything specific but tox4 has made many incompatible changes. Thanks for your work on this.","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"f6b2d48bbce2dd0f1e930ca27537964a80004b5d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"8456f45e_1412419f","updated":"2023-01-13 09:18:31.000000000","message":"Here [1] are the patches that check whether this change broke something in jobs that use run-tempest role. If a job failed it was not related to this change. \n\n[1] https://review.opendev.org/q/topic:test/updated-run-tempest-role","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"86d0f87a1872749760c5d5d6aaf7ce4aacda382f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"fc993701_e75d4bc0","updated":"2023-01-06 20:03:58.000000000","message":"recheck tox error should be fixed by now","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"cac1a94e0bdb738f11a15059fe49df7b58dea9b6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"a43b7f20_c5bcd814","updated":"2023-01-09 10:09:34.000000000","message":"thanks for fixing this Lukáš","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"394e0fe8237e2f165a90f36cabe27ed8e10ece64","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"3dd68c54_91959c99","updated":"2023-01-10 19:33:57.000000000","message":"what are the job failing on that? I am worried whether it is backward compatible or not. We use tox\u003c4 on stable branch testing and these roles form tempest will be used for stable branch jobs also. Can you also add some stable branches testing DNM patches (like nova, cinder and the one passing empty string) to verify the behavior. Till than procedural -1 to avoid accidental merge. ","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"01fb82a01eecba77225c4a1c0f05d7ba3316385b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"f252ba26_38654c44","in_reply_to":"07c6f24d_1c0e5295","updated":"2023-01-11 10:59:45.000000000","message":"Ok, it makes sense to check that it won\u0027t break other jobs first. I created the DNM patches here:\ncinder: https://review.opendev.org/c/openstack/cinder/+/869805\nnova: https://review.opendev.org/c/openstack/nova/+/869804\npython-tempestconf: https://review.opendev.org/c/openinfra/python-tempestconf/+/846108 (here is an example of what this patch fixes -- as Martin mentioned patches #5 and #6)\n\nThe issue is that the newer tox version is probably parsing empty string differently [1].\n\n[1] https://paste.openstack.org/show/bJf4AjBamnAVYrcPEYMQ/","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"1974ba46fc154fbfb32e39b1b54a583e0d0d9955","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"07c6f24d_1c0e5295","in_reply_to":"3dd68c54_91959c99","updated":"2023-01-10 22:29:36.000000000","message":"we noticed it in a few jobs on python-tempestconf\u0027s side:\nhttps://review.opendev.org/c/openinfra/python-tempestconf/+/846108\nSee the job failures in patchset #5, e.g.: https://zuul.opendev.org/t/openstack/build/2cea65015ae0499a925fea42b99c4e9b , patchset #6 depends on this patch in tempest and we no longer see the failure - although the jobs failed again on something else, one component\u0027s tests are getting 500 response :/ not related to this i believe","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"52d658147fb6262f2001217d42d12ee0bdaeae8b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"c18ab560_428aee58","in_reply_to":"c6e404d9_b53178bf","updated":"2023-01-19 10:33:25.000000000","message":"I think that in both cases the tempest arg contains the empty string. The difference is that in master job tox\u003d\u003d3.28.0 is installed and in the stable branch jobs tox\u003d\u003d4.2.3 is used. And the failure occurred only in the stable branch jobs because tox treats empty string differently with the new version [1].\n\nIn the master job the older version of tox is installed by this step [2] during the installation of devstack.\n\n[1] https://paste.openstack.org/show/bJf4AjBamnAVYrcPEYMQ/\n[2] https://opendev.org/openstack/devstack/src/commit/4356a64a0075eef6281e01467ad28265a790f966/lib/neutron_plugins/ovn_agent#L355","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"8c0f14e91f0e1ff7f0886851f6996346531f55ae","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"c6e404d9_b53178bf","in_reply_to":"f252ba26_38654c44","updated":"2023-01-18 19:45:11.000000000","message":"I see in PS5 where master job was passing fine but stable jobs are failing, I did not find any difference in the tempest run arg, do you know why only stable branch job tempest arg were considered invalid (or say passing empty string )?\n\npython-tempestconf-tempest-devstack-admin","commit_id":"5fec2510cbaac3e16f6855e4ed5aa2ce17dc35e4"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"5c7219eb3b6828f14af1e78c05bc9cb89867165a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"2debeccb_eb2c5c90","updated":"2023-02-02 03:08:47.000000000","message":"I debugged it and I do not think tox4 syntax change is any issue here.\n\nAll tempest jobs running on master gate use tox4 and pass. I think (including plugins jobs) we have all the combination of passing and not passing the tempest regex, black regex etc\n\nFor example, tempest-full-py3 job (if you expand the run-tempest role from console https://zuul.opendev.org/t/openstack/build/2cea65015ae0499a925fea42b99c4e9b/console)\n\nIt pass tempest_test_regex as \"\" and below is the rin-tempest role command is:\n\n      \"tox\" \n      \"-e\" \n      \"full\"\n      \"--\" \n      \"\"\n      \"--concurrency\u003d4\"\n\nand command from tox to tempest run is:\n\n      tempest run --regex \u0027(?!.*\\[.*\\bslow\\b.*\\])(^tempest\\.api)\u0027 --concurrency\u003d4\n\nanother example for stable/victoria run is https://zuul.opendev.org/t/openstack/build/86f9c60b0ded4e7e99799ee7042c60d4/console\n\nIn both places, the run-tempest role works fine with all tox (\u003c4 in stable/victoria and \u003e\u003d4 on master).\n\nNow coming back to python-tempestconf failure in victoria job. stabl;e/victoria job of python-tempestconf use run-tempest-26 role\n-  https://opendev.org/openinfra/python-tempestconf/src/commit/6488be3e8294ddceea0028b208bdcda791dc0a67/playbooks/python-tempestconf-tempest-devstack.yaml#L64\n\nand run-tempest-26 seems out of dated to handle the old/new regex things and it fail. With zuul inventory, all stable jobs including stable/train to stable/victoria pick \u0027run-tempest\u0027 roles\nexample:\n stable/victoria: https://zuul.opendev.org/t/openstack/build/86f9c60b0ded4e7e99799ee7042c60d4/console\n stable/train: https://zuul.opendev.org/t/openstack/build/650d9a6f17cd453aa1fca57fd4d7bf09/console\n\nAnd that is why I pushed the fix in run-tempest role - https://review.opendev.org/c/openstack/tempest/+/822339\n\nFix can be python-tempestconf  can do the same as any other job on stable/victoria doing - https://review.opendev.org/c/openinfra/python-tempestconf/+/872486","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"150504441bb77eaeb10ce3d63d4d712b23f3b95a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"4f902d70_52054d4b","updated":"2023-02-02 19:41:18.000000000","message":"I think I am convinced now and this is another example of tox4 breaking change and not sure what is the rational behind doing it.\n\nThanks for being patient with me here. nice fix.","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"d8170d5cc5385d3e0eba61d05675791ba48de7b4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"f17f4fc0_0c002230","updated":"2023-02-06 10:45:30.000000000","message":"recheck\nnova-live-migration failed with an error that is not caused by this patch.\n","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"5e40b47c363771018fe80984eebb2518cec05bd8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"030322e3_32e7e012","updated":"2023-02-03 11:47:55.000000000","message":"recheck \nmultinode error","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"6d9ffc7e9d3819c587e6076828f1b3d187be4a8e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"7f5b76f8_96f36277","updated":"2023-02-03 07:21:10.000000000","message":"recheck multinode failed","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"daff3d76c31fea11745c53206b774f4bb850a8e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"b073cdb3_39b352bf","updated":"2023-02-03 15:41:13.000000000","message":"recheck multinode unstable plus random timeout on other jobs","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"1ad1b9ff556d9fae27b9af5081adfb00420d3bc8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"f547d69d_149fbc05","updated":"2023-02-05 21:13:32.000000000","message":"recheck weird timeout out in zed job","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"00ce8bbe99190d3c57a088544aa5fba7816e4598","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"a20ee9b6_0daa6490","in_reply_to":"2debeccb_eb2c5c90","updated":"2023-02-02 11:04:01.000000000","message":"I am not 100 % sure about it but I think that the tempest-full-py3 on master runs tempest tests with tox\u003d\u003d3.28.0 [1] (look for `run-tempest : Check Tox version`)? I think that if it used tox\u003e\u003d4.0.0 then the job would behave a bit differently (it would probably fail the same way as the python-tempestconf job).\n\n[1] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_07a/872515/1/check/tempest-full-py3/07a5bbc/job-output.txt","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"2fc06d90816922f25be88dc6be87e3376bee7e09","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"d94214ae_56ebae20","in_reply_to":"2debeccb_eb2c5c90","updated":"2023-02-02 08:22:52.000000000","message":"victoria, wallaby and xena jobs will be removed from python-tempestconf once the gates are fixed - https://review.opendev.org/c/openinfra/python-tempestconf/+/865570\n\nIn your patch, https://review.opendev.org/c/openinfra/python-tempestconf/+/872486, yoga job failed (https://zuul.opendev.org/t/openstack/build/275792525059477fb305f7ac3ce044c1) on:\n```\ntempest run: error: unrecognized arguments: \nsmoke: exit 2 (1.99 seconds) /opt/stack/tempest\u003e tempest run --regex \u0027[.*\\bsmoke\\b.*]\u0027 \u0027\u0027 --concurrency\u003d2 \n```\nwhich is the error this tempest patch addresses","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":30674,"name":"Lukas Piwowarski","email":"lpiwowar@redhat.com","username":"lukas-piwowarski"},"change_message_id":"9fa2bdd5fdc8ccdfc3ccd3e13bef27104bfea641","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"051934a6_f91bb1b4","in_reply_to":"4f902d70_52054d4b","updated":"2023-02-03 09:30:55.000000000","message":"thanks!","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"2fd3212502ffc939ab286dde2e630d357ad6c17b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"e7c024a4_feea01c5","in_reply_to":"5a7771d2_fb16a62f","updated":"2023-02-02 19:08:32.000000000","message":"yes, it is failing with run-tempest role also, I thought its run-tempest-26 role issue only.\n\n@Martin, right, stable/yoga end up using the tox\u003e4 and master and stable/xena still using tox\u003c4 which is strange and not sure who is bumping tox in stable/yoga (that is another issue you need to fix in tempestconf)\n\n- stable/yoga using tox\u003e4  https://zuul.opendev.org/t/openstack/build/46778f7dc92a4d629003961761eda028/log/controller/logs/pip3-freeze.txt#250\n\n- stable/xena using tox\u003c4 https://zuul.opendev.org/t/openstack/build/7acf9c1148fb4b93b1ae1f39e1a2311d/log/controller/logs/pip3-freeze.txt#253\n\n- master admin jobs using tox\u003c4 https://zuul.opendev.org/t/openstack/build/865935a404df4266ba80a2a4922b2111/log/controller/logs/pip3-freeze.txt#246","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"de1130832d4535307faa4fef5bc4f561f68a5b92","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5a7771d2_fb16a62f","in_reply_to":"a20ee9b6_0daa6490","updated":"2023-02-02 18:57:27.000000000","message":"ah right. I pinned it on master and completly forget about it https://review.opendev.org/c/openstack/devstack/+/867065/2/lib/tempest#782","commit_id":"bd6481d93431738c72964ea0f8dd1d3e40c9778a"}]}