)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"7bce57e33087a5a73c9378c8ed6efa094b3d01cc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"842b7f57_bc18ffca","updated":"2022-10-13 13:37:33.000000000","message":"Thank you Alfredo! I am afraid this will not test aoc 2.0.0 preview/prerelease though, as explained in the comments","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"}],"quickstart.sh":[{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"7bce57e33087a5a73c9378c8ed6efa094b3d01cc","unresolved":true,"context_lines":[{"line_number":142,"context_line":"    if [[ -d ~/src/opendev.org/openstack/ansible-collections-openstack ]]; then"},{"line_number":143,"context_line":"        # Install zuul-cloned repo"},{"line_number":144,"context_line":"        pushd ~/src/opendev.org/openstack/ansible-collections-openstack"},{"line_number":145,"context_line":"        git checkout $ACO_BRANCH"},{"line_number":146,"context_line":"        ansible-galaxy collection install --force . \\"},{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"}],"source_content_type":"text/x-sh","patch_set":2,"id":"1589f946_8e4161a0","line":145,"updated":"2022-10-13 13:37:33.000000000","message":"This will pull the latest code from aoc\u0027s ACO_BRANCH branch and not what is checked out in the aoc repo, thus e.g. breaking our tripleo jobs in aoc.","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"05722ef7c0cb1ff7e9c77968dfe6a6f088443703","unresolved":true,"context_lines":[{"line_number":142,"context_line":"    if [[ -d ~/src/opendev.org/openstack/ansible-collections-openstack ]]; then"},{"line_number":143,"context_line":"        # Install zuul-cloned repo"},{"line_number":144,"context_line":"        pushd ~/src/opendev.org/openstack/ansible-collections-openstack"},{"line_number":145,"context_line":"        git checkout $ACO_BRANCH"},{"line_number":146,"context_line":"        ansible-galaxy collection install --force . \\"},{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"}],"source_content_type":"text/x-sh","patch_set":2,"id":"e6258ecb_a267571d","line":145,"in_reply_to":"1589f946_8e4161a0","updated":"2022-10-13 13:38:50.000000000","message":"\u0027pull\u0027 \u003d\u003e \u0027checkout\u0027","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"7bce57e33087a5a73c9378c8ed6efa094b3d01cc","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"2b90833c_f9d86342","line":150,"updated":"2022-10-13 13:37:33.000000000","message":"This will pull the latest code from ACO_BRANCH and this will then shadow the aoc rpm which gets installed later (*). So our tripleo ci jobs will not test whether the rpm is working, instead it will test whether the latest code in aoc upstream is working with rdo.\n\n(*) ansible-galaxy collection install will install aoc to ~/.ansible/collections which is preferred by ansible over the /usr/... path where rdo rpms gets installed.","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":8833,"name":"Rabi Mishra","email":"ramishra@redhat.com","username":"rabi"},"change_message_id":"09ac0283e95ac3b34b9214c75ccbbbf8402c1624","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"2934474b_dca78217","line":150,"in_reply_to":"003eeafe_36c9f67d","updated":"2022-10-14 10:24:16.000000000","message":"Though I think tripleoclient won\u0027t use the galaxy installed rpms in  $VIRTUAL_ENV/share/ansible/collections, I\u0027ve pushed a WIP patch[1] to avoid using ~/.ansible as well, let\u0027s see.\n\n[1] https://review.opendev.org/c/openstack/python-tripleoclient/+/861354","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":8833,"name":"Rabi Mishra","email":"ramishra@redhat.com","username":"rabi"},"change_message_id":"a7ba4cf8ee5badd072be82445466009bc2991c45","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"49325194_87822c7c","line":150,"in_reply_to":"2b90833c_f9d86342","updated":"2022-10-14 06:53:49.000000000","message":"\u003e ansible-galaxy collection install will install aoc to ~/.ansible/collections\n\nIsn\u0027t -p option to provide the location of install. Also, can we not override COLLECTIONS_PATHS to use collections from specific locations?\n\nBtw, There is only one instance of direct AOC module usage in tq/tqe, though the os_tempest role used for tempest uses a few of them too[1]\n\n[1] https://opendev.org/openstack/openstack-ansible-os_tempest/src/branch/master/tasks/tempest_resources.yml#L180","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"25ea3af70731e1af9bbf04534df43e39ed006bb3","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"003eeafe_36c9f67d","line":150,"in_reply_to":"3328fc68_e52573fe","updated":"2022-10-14 09:55:09.000000000","message":"The reason why role os_tempest is not using the RPM is that the role is run by the Ansible instance that quickstart starts [1],[2]. When this instance is started, collections used by this instance have to be available.\n\n[1] https://opendev.org/openstack/tripleo-quickstart-extras/src/commit/048667a14d20278266a17f7e1f5e23bc74157aac/playbooks/quickstart-extras-validate.yml#L24\n[2] https://opendev.org/openstack/tripleo-quickstart-extras/src/commit/048667a14d20278266a17f7e1f5e23bc74157aac/playbooks/tasks/tempest.yml#L275","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":8833,"name":"Rabi Mishra","email":"ramishra@redhat.com","username":"rabi"},"change_message_id":"44265f9e5df5e9ce39e3d13d0bc389bb02258a7f","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"b9e47a24_48d5c429","line":150,"in_reply_to":"49325194_87822c7c","updated":"2022-10-14 07:18:06.000000000","message":"From https://review.opendev.org/c/openstack/tripleo-quickstart/+/861278 failures I can see that tq/tqe use the collections installed from galaxy in the workspace and not from the rpm. So not sure why this will break anything and why?","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"9e96e4d5cc2ab66ac37ac4698949c775223a9525","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"d9bad5d3_7d12c920","line":150,"in_reply_to":"b9e47a24_48d5c429","updated":"2022-10-14 08:45:00.000000000","message":"Yes, Ansible will use the OpenStack collection (AOC) from Galaxy and ignore the RPM. This means, the jobs will test TripleO deployments against Galaxy releases but not against RDO\u0027s RPMs for AOC. RDO users and downstream releases dont use quickstart, so they will use e.g. the RPMs. Since we dont test the RPMs in our CI, we dont know whether they actually work.\n\nThis is not a new issue though. So far, we pinned the AOC release we pull from Galaxy to the exact same version that Alfredo packages in RDO in order to get the same content. This is not maintainable though, we already drifted apart, i.e. RDO has AOC 1.9.1 while quickstart still has 1.8.0.\n\nWe thought about overriding COLLECTIONS_PATHS but one thing we have to clarify here first is that the Ansible instance that quickstart launches is NOT the same Ansible instance that deploys TripleO, else COLLECTIONS_PATHS wont have any positive effect.\nRabi, maybe you can give us some insights here? Or point us to code where we can check that or point us to someone we could ask?","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":8833,"name":"Rabi Mishra","email":"ramishra@redhat.com","username":"rabi"},"change_message_id":"cc1179da05eed19b884a4f70dad03a05fde6d25f","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"29d2d4d9_d269a87e","line":150,"in_reply_to":"d89fae04_4a3acce4","updated":"2022-10-14 09:53:58.000000000","message":"Also I think tripleoclient is using rpm collections with this patch and not the galaxy installed ones in CI.","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":32962,"name":"Jakob Meng","email":"code@jakobmeng.de","username":"jakobmeng@web.de"},"change_message_id":"5e6730ff1615c3aaf560b85512fcae8bc4a4af36","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"3328fc68_e52573fe","line":150,"in_reply_to":"d89fae04_4a3acce4","updated":"2022-10-14 09:50:19.000000000","message":"Went down the rabbit hole...\n\nJob tripleo-ci-centos-9-ovb-3ctlr_1comp-featureset001 [1] uses Ansible\u0027s shell module for \u0027undercloud install\u0027 [2].\n\nJob tripleo-ci-centos-9-standalone [3] uses Ansible\u0027s shell module for \u0027Standalone deploy\u0027 [4].\n\nAnsible\u0027s shell module is actually a action plugin [5] which calls Ansible\u0027s command module [6] which then calls Ansible\u0027s run_command function from the AnsibleModule base class [7]. This run_command will use the environment variables of the Ansible instance running this module and uses them for the actual shell execution [8].\n\nDefining Ansible\u0027s ANSIBLE_COLLECTIONS_PATHS variable will (probably) passed down to the TripleO deployment commands. But will it be passed further down to the Ansible instances later in the deployment process? Somebody more familiar with TripleO will have to answer.\n\nIn case it is, we definitely want to modify ANSIBLE_COLLECTIONS_PATHS in later tripleo stages.\n\n[1] https://logserver.rdoproject.org/70/861170/2/openstack-check/tripleo-ci-centos-9-ovb-3ctlr_1comp-featureset001/3e173dc/job-output.txt\n[2] https://opendev.org/openstack/tripleo-operator-ansible/src/commit/e4b46582c3780e1b8fcfe347b8342c41548cab17/roles/tripleo_undercloud_install/tasks/main.yml#L40\n[3] https://1ba0cb977f32e92b0d03-59d1b57b6b0a10f841577c6847231725.ssl.cf1.rackcdn.com/861170/2/check/tripleo-ci-centos-9-standalone/9406e01/job-output.txt\n[4] https://opendev.org/openstack/tripleo-operator-ansible/src/commit/e4b46582c3780e1b8fcfe347b8342c41548cab17/roles/tripleo_deploy/tasks/main.yml#L89\n[5] https://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/action/shell.py\n[6] https://github.com/ansible/ansible/blob/devel/lib/ansible/modules/command.py\n[7] https://github.com/ansible/ansible/blob/58f051a9414bde2ac61d3b9ac2e88e1cf9e1e7dc/lib/ansible/module_utils/basic.py#L1841\n[8] https://github.com/ansible/ansible/blob/58f051a9414bde2ac61d3b9ac2e88e1cf9e1e7dc/lib/ansible/module_utils/basic.py#L1963","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"},{"author":{"_account_id":8833,"name":"Rabi Mishra","email":"ramishra@redhat.com","username":"rabi"},"change_message_id":"f1e64e69da0bd3689bf28cc8f4ade76ba9da0f7a","unresolved":true,"context_lines":[{"line_number":147,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":148,"context_line":"        popd"},{"line_number":149,"context_line":"    else"},{"line_number":150,"context_line":"        ansible-galaxy collection install --force \\"},{"line_number":151,"context_line":"            git+https://opendev.org/openstack/ansible-collections-openstack,$ACO_BRANCH \\"},{"line_number":152,"context_line":"            -p $VIRTUAL_ENV/share/ansible/collections"},{"line_number":153,"context_line":"    fi"}],"source_content_type":"text/x-sh","patch_set":2,"id":"d89fae04_4a3acce4","line":150,"in_reply_to":"d9bad5d3_7d12c920","updated":"2022-10-14 09:42:31.000000000","message":"tripleoclient uses ansible-runner to run all playbooks. We can set the ANSIBLE_COLLECTIONS_PATHS there[1]. I also noticed that we set collections path[2] for CI which we can probably change if needed. But what is confusing to me is why the rpm installed collections are not used by os_tempest[3] when running tqe playbooks with [2] set.\n\n\n\n[1] https://github.com/openstack/python-tripleoclient/blob/master/tripleoclient/utils.py#L487\n[2] https://opendev.org/openstack/tripleo-ci/src/branch/master/roles/run-test/templates/toci_quickstart.sh.j2#L131\n[3] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_977/861278/1/check/tripleo-ci-centos-9-containers-multinode/9778c9c/logs/quickstart_install.log","commit_id":"1995a1ad6450f21cd73ec1fd9124ad19af379a36"}]}