)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"a82f997150c4a4f64e5bc4d17f1eec89e9a4dcc7","unresolved":true,"context_lines":[{"line_number":10,"context_line":"included tests, making tempest fails. This doesn\u0027t means the whole build"},{"line_number":11,"context_line":"have issues, only that tempest wasn\u0027t able to find tests to be executed."},{"line_number":12,"context_line":"Since os_tempest is being used in other projects, and those projects do"},{"line_number":13,"context_line":"not run all tempest tests due the resources, and very often the list of"},{"line_number":14,"context_line":"included and excluded matches, it\u0027s a good idea in those cases, to have"},{"line_number":15,"context_line":"a test that will for sure be executed, so it won\u0027t block the gates due"},{"line_number":16,"context_line":"to a false failure."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"89d13c4f_4a660a51","line":14,"range":{"start_line":13,"start_character":49,"end_line":14,"end_character":29},"updated":"2021-06-17 10:51:24.000000000","message":"I don\u0027t quite understand how this may happen?\n\nWhy not to override default tempest_test_default_includelist and explicitly define set of tests you want to run instead of placing super simmilar excludelist?\n\nIe we\u0027re including all magnum tests except some specific ones:\nhttps://opendev.org/openstack/openstack-ansible/src/branch/master/tests/roles/bootstrap-host/templates/user_variables_magnum.yml.j2#L54-L64","commit_id":"cc3726f6b80ac011e38aa0eef421cb0995cc0ee0"},{"author":{"_account_id":8367,"name":"Arx Cruz","email":"arxcruz@redhat.com","username":"arxcruz"},"change_message_id":"b3ca035c4aa9824b648273b7a0341c1587a2d75b","unresolved":true,"context_lines":[{"line_number":10,"context_line":"included tests, making tempest fails. This doesn\u0027t means the whole build"},{"line_number":11,"context_line":"have issues, only that tempest wasn\u0027t able to find tests to be executed."},{"line_number":12,"context_line":"Since os_tempest is being used in other projects, and those projects do"},{"line_number":13,"context_line":"not run all tempest tests due the resources, and very often the list of"},{"line_number":14,"context_line":"included and excluded matches, it\u0027s a good idea in those cases, to have"},{"line_number":15,"context_line":"a test that will for sure be executed, so it won\u0027t block the gates due"},{"line_number":16,"context_line":"to a false failure."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"f661d7bf_1613a251","line":14,"range":{"start_line":13,"start_character":49,"end_line":14,"end_character":29},"in_reply_to":"89d13c4f_4a660a51","updated":"2021-06-17 11:04:43.000000000","message":"An example, some jobs we are running only tempest.scenario.TestBasicNetworkScenario that contains like 3 tests, and these tests are failing due some known bug. We add it in our skiplist [1]. So, at the end, all tests under TestBasicNetworkScenario are being skipped, and our jobs are failing because of that. We can\u0027t afford to run more than a few tests on check and gate because of the time/resources consuming.\n\n1 - https://opendev.org/openstack/openstack-tempest-skiplist/src/branch/master/roles/validate-tempest/vars/tempest_skip.yml","commit_id":"cc3726f6b80ac011e38aa0eef421cb0995cc0ee0"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"b96c171cdccd7702d9cb50e80e5a743b7745e9bf","unresolved":true,"context_lines":[{"line_number":10,"context_line":"included tests, making tempest fails. This doesn\u0027t means the whole build"},{"line_number":11,"context_line":"have issues, only that tempest wasn\u0027t able to find tests to be executed."},{"line_number":12,"context_line":"Since os_tempest is being used in other projects, and those projects do"},{"line_number":13,"context_line":"not run all tempest tests due the resources, and very often the list of"},{"line_number":14,"context_line":"included and excluded matches, it\u0027s a good idea in those cases, to have"},{"line_number":15,"context_line":"a test that will for sure be executed, so it won\u0027t block the gates due"},{"line_number":16,"context_line":"to a false failure."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"12367047_4ab032df","line":14,"range":{"start_line":13,"start_character":49,"end_line":14,"end_character":29},"in_reply_to":"f661d7bf_1613a251","updated":"2021-06-17 14:00:30.000000000","message":"Well, IMO, but that would be a valid CI failure, because you don\u0027t actually run relevant tests.\nSo instead of doing smth with test results or define another set of tests explicitly, you\u0027re suggesting to run fake one and even not knowing that you\u0027re not running anything useful?","commit_id":"cc3726f6b80ac011e38aa0eef421cb0995cc0ee0"},{"author":{"_account_id":25023,"name":"Jonathan Rosser","email":"jonathan.rosser@rd.bbc.co.uk","username":"jrosser"},"change_message_id":"d9fa37c1ed52a45b7214c1e01467bc6ac8dd0c9d","unresolved":true,"context_lines":[{"line_number":11,"context_line":"have issues, only that tempest wasn\u0027t able to find tests to be executed."},{"line_number":12,"context_line":"Since os_tempest is being used in other projects, and those projects do"},{"line_number":13,"context_line":"not run all tempest tests due the resources, and very often the list of"},{"line_number":14,"context_line":"included and excluded matches, it\u0027s a good idea in those cases, to have"},{"line_number":15,"context_line":"a test that will for sure be executed, so it won\u0027t block the gates due"},{"line_number":16,"context_line":"to a false failure."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"This patch adds the tempest_test_alternative_test variable and create a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"f36301c1_7eda6eac","line":15,"range":{"start_line":14,"start_character":31,"end_line":15,"end_character":37},"updated":"2021-06-17 14:10:30.000000000","message":"an alternative would be to unconditionally add the keystone tests to the list of included tests supplied to this role? Seems like a lot of code and some duplication is introduced here to patch over an unusual set of input.","commit_id":"2147510a8c79a8db235a5ed162738a22c2abb6c5"}],"defaults/main.yml":[{"author":{"_account_id":25023,"name":"Jonathan Rosser","email":"jonathan.rosser@rd.bbc.co.uk","username":"jrosser"},"change_message_id":"d9fa37c1ed52a45b7214c1e01467bc6ac8dd0c9d","unresolved":true,"context_lines":[{"line_number":95,"context_line":"# This sets up a list of tests to execute based on what\u0027s deployed in the environment."},{"line_number":96,"context_line":"# The list gets added to the includelist which tempest executes."},{"line_number":97,"context_line":"# Defaults to tempest_test_whitelist for backwards compatibility and migration purposes."},{"line_number":98,"context_line":"tempest_test_includelist: \"{{ tempest_test_whitelist | default(tempest_test_default_includelist) }}\""},{"line_number":99,"context_line":"tempest_test_default_includelist:"},{"line_number":100,"context_line":"  - \"smoke\""},{"line_number":101,"context_line":"  - \"{{ (tempest_service_available_ceilometer | bool) | ternary(\u0027tempest.api.telemetry\u0027, \u0027\u0027) }}\""}],"source_content_type":"text/x-yaml","patch_set":2,"id":"5ebe9ed6_3f477ea4","line":98,"range":{"start_line":98,"start_character":0,"end_line":98,"end_character":100},"updated":"2021-06-17 14:10:30.000000000","message":"another simpler approach might be to define a new variable `tempest_test_default_tests: \"keystone_tempest_plugin.tests.api\"` and concatenate it with tempest_test_default_includelist at this point, then no extra logic would be required in the tasks and it would be possible to disable that behaviour by overriding tempest_test_default_tests as an empty list if necessary.","commit_id":"2147510a8c79a8db235a5ed162738a22c2abb6c5"},{"author":{"_account_id":25023,"name":"Jonathan Rosser","email":"jonathan.rosser@rd.bbc.co.uk","username":"jrosser"},"change_message_id":"b09f9129d3d9f19bf3bcd59b71f6ec7e24d4bedd","unresolved":true,"context_lines":[{"line_number":95,"context_line":"# This sets up a list of tests to execute based on what\u0027s deployed in the environment."},{"line_number":96,"context_line":"# The list gets added to the includelist which tempest executes."},{"line_number":97,"context_line":"# Defaults to tempest_test_whitelist for backwards compatibility and migration purposes."},{"line_number":98,"context_line":"tempest_test_includelist: \"{{ tempest_test_whitelist | default(tempest_test_default_includelist) }}\""},{"line_number":99,"context_line":"tempest_test_default_includelist:"},{"line_number":100,"context_line":"  - \"smoke\""},{"line_number":101,"context_line":"  - \"{{ (tempest_service_available_ceilometer | bool) | ternary(\u0027tempest.api.telemetry\u0027, \u0027\u0027) }}\""}],"source_content_type":"text/x-yaml","patch_set":2,"id":"77640c3a_98825edc","line":98,"range":{"start_line":98,"start_character":0,"end_line":98,"end_character":100},"in_reply_to":"5ebe9ed6_3f477ea4","updated":"2021-06-17 14:14:06.000000000","message":"I do tend to agree with with dimitry though that this patch really does seem to be addressing the symptom of weird/broken input data rather than addressing the root cause","commit_id":"2147510a8c79a8db235a5ed162738a22c2abb6c5"}],"tasks/tempest_post_install.yml":[{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"26ee08aab25fa274db853139e3e662f16a8c19ec","unresolved":true,"context_lines":[{"line_number":160,"context_line":"- name: Insert aditional tests in the tempest_test_includelist"},{"line_number":161,"context_line":"  set_fact: "},{"line_number":162,"context_line":"    tempest_test_includelist: \"{{ tempest_test_includelist + [ tempest_test_extra_test ] }}\""},{"line_number":163,"context_line":"  when: tempest_test_extra_test is defined"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"- name: Generate tempest test include list"},{"line_number":166,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"9723d7f7_6f85058b","line":163,"range":{"start_line":163,"start_character":0,"end_line":163,"end_character":42},"updated":"2021-06-18 11:18:53.000000000","message":"it\u0027s always defined as it\u0027s inside defaults","commit_id":"e3bd0d30656199ad3b5a22eb455bb97c934cd49f"},{"author":{"_account_id":8367,"name":"Arx Cruz","email":"arxcruz@redhat.com","username":"arxcruz"},"change_message_id":"a0d416af7ba9fdf2bd22483f1ef3300265ed3d49","unresolved":true,"context_lines":[{"line_number":160,"context_line":"- name: Insert aditional tests in the tempest_test_includelist"},{"line_number":161,"context_line":"  set_fact: "},{"line_number":162,"context_line":"    tempest_test_includelist: \"{{ tempest_test_includelist + [ tempest_test_extra_test ] }}\""},{"line_number":163,"context_line":"  when: tempest_test_extra_test is defined"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"- name: Generate tempest test include list"},{"line_number":166,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"69d82ef5_eff7191d","line":163,"range":{"start_line":163,"start_character":0,"end_line":163,"end_character":42},"in_reply_to":"9723d7f7_6f85058b","updated":"2021-06-18 11:26:51.000000000","message":"I intent to not define it in the defaults, so we can set it on our side.","commit_id":"e3bd0d30656199ad3b5a22eb455bb97c934cd49f"},{"author":{"_account_id":25023,"name":"Jonathan Rosser","email":"jonathan.rosser@rd.bbc.co.uk","username":"jrosser"},"change_message_id":"9bb37b4bdad71f60e314c80fc038d1ddc2c0cfdb","unresolved":true,"context_lines":[{"line_number":158,"context_line":"# rid of all of it in all the places. Once we implement the include list in the"},{"line_number":159,"context_line":"# openstack-tempest-skiplist side, we can remove it"},{"line_number":160,"context_line":"- name: Insert aditional tests in the tempest_test_includelist"},{"line_number":161,"context_line":"  set_fact: "},{"line_number":162,"context_line":"    tempest_test_includelist: \"{{ tempest_test_includelist + [ tempest_test_extra_test ] }}\""},{"line_number":163,"context_line":"  when: tempest_test_extra_test is defined"},{"line_number":164,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":4,"id":"bf79f02f_c01c6710","line":161,"range":{"start_line":161,"start_character":11,"end_line":161,"end_character":12},"updated":"2021-06-21 09:24:40.000000000","message":"extra whitespace","commit_id":"c91b827595709bc6ae759b87817407fd1aa4c3da"},{"author":{"_account_id":8367,"name":"Arx Cruz","email":"arxcruz@redhat.com","username":"arxcruz"},"change_message_id":"6ba89b3616cf7bc175c13ce152c0d574f224a4f7","unresolved":false,"context_lines":[{"line_number":158,"context_line":"# rid of all of it in all the places. Once we implement the include list in the"},{"line_number":159,"context_line":"# openstack-tempest-skiplist side, we can remove it"},{"line_number":160,"context_line":"- name: Insert aditional tests in the tempest_test_includelist"},{"line_number":161,"context_line":"  set_fact: "},{"line_number":162,"context_line":"    tempest_test_includelist: \"{{ tempest_test_includelist + [ tempest_test_extra_test ] }}\""},{"line_number":163,"context_line":"  when: tempest_test_extra_test is defined"},{"line_number":164,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":4,"id":"4a3a6306_97f8befe","line":161,"range":{"start_line":161,"start_character":11,"end_line":161,"end_character":12},"in_reply_to":"bf79f02f_c01c6710","updated":"2021-06-21 10:40:04.000000000","message":"Done","commit_id":"c91b827595709bc6ae759b87817407fd1aa4c3da"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"25401b0e4bd8683a8d144864fb736d061b91c2da","unresolved":true,"context_lines":[{"line_number":165,"context_line":"- name: Generate tempest test include list"},{"line_number":166,"context_line":"  copy:"},{"line_number":167,"context_line":"    content: |"},{"line_number":168,"context_line":"      {% for item in tempest_test_includelist | unique | sort %}"},{"line_number":169,"context_line":"      {%   if item %}"},{"line_number":170,"context_line":"      {{ item }}"},{"line_number":171,"context_line":"      {%   endif %}"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"ad926e47_fbba23fc","line":168,"range":{"start_line":168,"start_character":21,"end_line":168,"end_character":45},"updated":"2021-06-22 06:41:26.000000000","message":"thinking about this whole thing one more time, I think it might be better to have:\n\n  {% for item in (tempest_test_includelist + tempest_test_extra_tests) | unique | sort %}\n\n\nand define tempest_test_extra_tests as empty list in defaults?","commit_id":"6c67e84dedc28ad6efef90c78f649decec1c927f"}],"tasks/tempest_run.yml":[{"author":{"_account_id":25023,"name":"Jonathan Rosser","email":"jonathan.rosser@rd.bbc.co.uk","username":"jrosser"},"change_message_id":"1ad53d5c9dba406d7f8a1b7534da9da487be0625","unresolved":true,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"  - name: Get content of test_list file"},{"line_number":37,"context_line":"    set_fact:"},{"line_number":38,"context_line":"      tempest_test_list: \"{{ lookup(\u0027file\u0027, \u0027{{ tempest_log_dir }}/test_list.txt\u0027) }}\""},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"  - name: Execute tempest tests"},{"line_number":41,"context_line":"    shell: |"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"415d37a3_ed42c38d","line":38,"range":{"start_line":38,"start_character":0,"end_line":38,"end_character":86},"updated":"2021-06-17 13:59:22.000000000","message":"as far as i understand, lookup executes on the ansible controller and the previous shell task that writes test_list.txt will run on whatever the ansible target of the task is. I don\u0027t think this is going to work when the ansible controller is not the same host as the target.","commit_id":"2147510a8c79a8db235a5ed162738a22c2abb6c5"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"25401b0e4bd8683a8d144864fb736d061b91c2da","unresolved":true,"context_lines":[{"line_number":42,"context_line":"    tags:"},{"line_number":43,"context_line":"      # don\u0027t trigger ANSIBLE0013"},{"line_number":44,"context_line":"      - skip_ansible_lint"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"  - name: Generate test-list file"},{"line_number":47,"context_line":"    shell: |"},{"line_number":48,"context_line":"      set -e"},{"line_number":49,"context_line":"      if [ -d {{ tempest_venv_bin }} ];"},{"line_number":50,"context_line":"      then"},{"line_number":51,"context_line":"      . {{ tempest_venv_bin }}/activate"},{"line_number":52,"context_line":"      fi"},{"line_number":53,"context_line":"      tempest run \\"},{"line_number":54,"context_line":"        --include-list {{ tempest_test_includelist_file_path }} \\"},{"line_number":55,"context_line":"      {% if tempest_test_excludelist | length \u003e 0 %}"},{"line_number":56,"context_line":"        --exclude-list {{ tempest_test_excludelist_file_path }} \\"},{"line_number":57,"context_line":"      {% endif %}"},{"line_number":58,"context_line":"        --list-tests \u0026\u003e {{ tempest_log_dir }}/test_list.txt"},{"line_number":59,"context_line":"    args:"},{"line_number":60,"context_line":"      chdir: \"{{ tempest_workspace }}\""},{"line_number":61,"context_line":"      executable: /bin/bash"},{"line_number":62,"context_line":"    changed_when: false"},{"line_number":63,"context_line":"    failed_when: false"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"  always:"},{"line_number":66,"context_line":"  - name: Generate raw subunit results"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"39f8ce60_74163290","line":63,"range":{"start_line":45,"start_character":0,"end_line":63,"end_character":22},"updated":"2021-06-22 06:41:26.000000000","message":"why we moved this block? We don\u0027t want test_list.txt to be generated when tempest fails? Why so?","commit_id":"6c67e84dedc28ad6efef90c78f649decec1c927f"}]}