)]}'
{".zuul.yaml":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b14a4e2adf0612f5f66f9dc5437b5c66fa3839ab","unresolved":true,"context_lines":[{"line_number":72,"context_line":"      - opendev.org/openstack/cinder-tempest-plugin"},{"line_number":73,"context_line":"      - opendev.org/openstack/cinder"},{"line_number":74,"context_line":"    vars:"},{"line_number":75,"context_line":"      tempest_concurrency: 3"},{"line_number":76,"context_line":"      tempest_test_regex: \u0027(^tempest\\.(api|scenario)|(^cinder_tempest_plugin))\u0027"},{"line_number":77,"context_line":"      tempest_test_exclude_list: \u0027{{ ansible_user_dir }}/{{ zuul.projects[\"opendev.org/openstack/tempest\"].src_dir }}/tools/tempest-integrated-gate-storage-exclude-list.txt\u0027"},{"line_number":78,"context_line":"      # Temporarily exclude TestMultiAttachVolumeSwap until LP bug #1980816 is resolved."}],"source_content_type":"text/x-yaml","patch_set":4,"id":"2d354691_63c8b4b4","line":75,"range":{"start_line":75,"start_character":6,"end_line":75,"end_character":28},"updated":"2023-08-11 18:11:16.000000000","message":"any particular reason or evidence that we need to run with 3 concurrency? it was 4 before where those jobs were passing or less failing ?","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"c6c91961b21807c62710df374f7f38ddb0218404","unresolved":true,"context_lines":[{"line_number":72,"context_line":"      - opendev.org/openstack/cinder-tempest-plugin"},{"line_number":73,"context_line":"      - opendev.org/openstack/cinder"},{"line_number":74,"context_line":"    vars:"},{"line_number":75,"context_line":"      tempest_concurrency: 3"},{"line_number":76,"context_line":"      tempest_test_regex: \u0027(^tempest\\.(api|scenario)|(^cinder_tempest_plugin))\u0027"},{"line_number":77,"context_line":"      tempest_test_exclude_list: \u0027{{ ansible_user_dir }}/{{ zuul.projects[\"opendev.org/openstack/tempest\"].src_dir }}/tools/tempest-integrated-gate-storage-exclude-list.txt\u0027"},{"line_number":78,"context_line":"      # Temporarily exclude TestMultiAttachVolumeSwap until LP bug #1980816 is resolved."}],"source_content_type":"text/x-yaml","patch_set":4,"id":"63c6555c_38736649","line":75,"range":{"start_line":75,"start_character":6,"end_line":75,"end_character":28},"in_reply_to":"2d354691_63c8b4b4","updated":"2023-08-11 19:20:41.000000000","message":"the error message of the failing jobs seems to hint about resource issues. \n\nNo, it was 6, not 4.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"654b090a025fd2e79e93b8efb5e537eb31e763f2","unresolved":true,"context_lines":[{"line_number":72,"context_line":"      - opendev.org/openstack/cinder-tempest-plugin"},{"line_number":73,"context_line":"      - opendev.org/openstack/cinder"},{"line_number":74,"context_line":"    vars:"},{"line_number":75,"context_line":"      tempest_concurrency: 3"},{"line_number":76,"context_line":"      tempest_test_regex: \u0027(^tempest\\.(api|scenario)|(^cinder_tempest_plugin))\u0027"},{"line_number":77,"context_line":"      tempest_test_exclude_list: \u0027{{ ansible_user_dir }}/{{ zuul.projects[\"opendev.org/openstack/tempest\"].src_dir }}/tools/tempest-integrated-gate-storage-exclude-list.txt\u0027"},{"line_number":78,"context_line":"      # Temporarily exclude TestMultiAttachVolumeSwap until LP bug #1980816 is resolved."}],"source_content_type":"text/x-yaml","patch_set":4,"id":"d86e9b30_f3c5a82d","line":75,"range":{"start_line":75,"start_character":6,"end_line":75,"end_character":28},"in_reply_to":"63c6555c_38736649","updated":"2023-09-07 16:39:32.000000000","message":"it depends on node you are running the job and tempest calculate the concurrency accordingly. \n\n- if num of CPU are \u003c3 then concurrency will be 3 \n- if num of CPU are \u003c3 then concurrency will be num of cpu -2. most the node in our CI is 8 cou so 6 is concurrency","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"3a34df9ce289813b0c41f6c5d61d195c0fb9c45b","unresolved":true,"context_lines":[{"line_number":72,"context_line":"      - opendev.org/openstack/cinder-tempest-plugin"},{"line_number":73,"context_line":"      - opendev.org/openstack/cinder"},{"line_number":74,"context_line":"    vars:"},{"line_number":75,"context_line":"      tempest_concurrency: 3"},{"line_number":76,"context_line":"      tempest_test_regex: \u0027(^tempest\\.(api|scenario)|(^cinder_tempest_plugin))\u0027"},{"line_number":77,"context_line":"      tempest_test_exclude_list: \u0027{{ ansible_user_dir }}/{{ zuul.projects[\"opendev.org/openstack/tempest\"].src_dir }}/tools/tempest-integrated-gate-storage-exclude-list.txt\u0027"},{"line_number":78,"context_line":"      # Temporarily exclude TestMultiAttachVolumeSwap until LP bug #1980816 is resolved."}],"source_content_type":"text/x-yaml","patch_set":4,"id":"5edf7289_db1d591f","line":75,"range":{"start_line":75,"start_character":6,"end_line":75,"end_character":28},"in_reply_to":"d86e9b30_f3c5a82d","updated":"2023-09-07 16:51:41.000000000","message":"I\u0027m aware of the code which sets the concurrency level, but all the failures I could see had concurrency 6 and I think it\u0027s a bit too much for the resources of those instances.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"}],"/COMMIT_MSG":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b14a4e2adf0612f5f66f9dc5437b5c66fa3839ab","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5956c426_ccf61bb1","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"updated":"2023-08-11 18:11:16.000000000","message":"it will be good to add information what failure they are failing and increased concurrency has caused them or more frequently (by comparing the failure before and after concurrency increase.)","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"1d1148c1b7d1e0a4d7efc734c4535cc56e708616","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"8805bde0_6c5f6557","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"in_reply_to":"4f52b231_01bcfb30","updated":"2023-09-07 17:24:24.000000000","message":"so you mean 6 concurrency unhide the more failures? \n\nSee it is very easy to adjust the concurrency and other testing config to make job pass but we really encourage to debug the things and fix so that we can have stable gate. TC has been working and encouraging/discussing it in almost every meeting. \n\nI will be more ok/comfirtable to this change if we can figure out/confirm that \"extra failure are introduced by the increase in concurrency and our CI infra resource limitation. Those are not actual bugs and cannot happen in production\"\n\nOtherwise we keep adjusting our testing bits and make job pass and hide actual bugs happening frequently in our CI as well as in production.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"c6c91961b21807c62710df374f7f38ddb0218404","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"9ce4f6dd_2a186793","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"in_reply_to":"5956c426_ccf61bb1","updated":"2023-08-11 19:20:41.000000000","message":"I don\u0027t have such information or specific information, and to be honest running with concurrency 6 definitely causes resource issues with such small deployments.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"3a34df9ce289813b0c41f6c5d61d195c0fb9c45b","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"4f52b231_01bcfb30","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"in_reply_to":"70d8ca6b_5e018c22","updated":"2023-09-07 16:51:41.000000000","message":"Again, there may be various failure but they spiked when the tempest concurrency was increased to 6.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"975bacdd76c881dc221664c3e638f2b669799439","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"93ae91e2_ae7d4944","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"in_reply_to":"8805bde0_6c5f6557","updated":"2023-09-07 18:06:08.000000000","message":"Regardless of the bugs, these instances can\u0027t likely handle that concurrency for certain tests we have in cinder-tempest-plugins.\nThis is not hiding things under the carpet, but just restoring a more sane usage of the resources.\nOtherwise we will likely not merge anything anymore.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"654b090a025fd2e79e93b8efb5e537eb31e763f2","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"The lvm-lio job family, the cbak-ceph job and"},{"line_number":10,"context_line":"the lvm-multiattach job (especially the latter)"},{"line_number":11,"context_line":"fail very frequently."},{"line_number":12,"context_line":"They used to run with --concurrency 6 on the gates,"},{"line_number":13,"context_line":"a lower concurrency level can help with resource issues."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I108b60c289acace9e17fa9cb78f84489989d5171"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"70d8ca6b_5e018c22","line":13,"range":{"start_line":11,"start_character":0,"end_line":13,"end_character":56},"in_reply_to":"9ce4f6dd_2a186793","updated":"2023-09-07 16:39:32.000000000","message":"Yea, we need to dig into those failure I think which are failing since more than year or so.\n\ncommented above the calculation of concurrency in tempest, it is not 6 always instead calculated as per the num of cou in the server you are tempest on","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"b14a4e2adf0612f5f66f9dc5437b5c66fa3839ab","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"a44b664d_675b7f23","updated":"2023-08-11 18:11:16.000000000","message":"I am still seeing those job failing and one timeout for which I increased the concurrency. I will say we need to dig into the actual reason for failure.","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"ede9e9038ecd324ca0cc882e213f975a56b654d5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"4bf5e234_a28be633","updated":"2023-09-07 16:33:13.000000000","message":"recheck\n\nfresh results, but in fact only cinder-tempest-plugin-lvm-multiattach failed","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"0bc10e7aad9805d52506bd84bdbacc7fdc21adb4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"77d95463_e9b2ce6d","updated":"2023-08-17 14:49:23.000000000","message":"recheck\n\njust two connection timeouts on cinder-tempest-plugin-lvm-lio-barbican","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"0f8bab1a523b9a4da3687231b494076d785619b8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"9b56f31c_5e161630","updated":"2023-10-24 15:03:50.000000000","message":"recheck for fresh logs","commit_id":"d6ad20cb29d407f3a0274e682378777a55b91b36"}]}